Files
kubernetes/pkg/registry/flowcontrol/ensurer/flowschema.go
Patrick Ohly 5e1c6cd0d4 pkg/registry/flowcontrol: avoid race condition during Create
k8s.io/kubernetes/test/integration/controlplane.TestReconcilerAPIServerLeaseMultiCombined
suffered from race conditions. The underlying reason is that
330b5a2b8d/staging/src/k8s.io/apimachinery/pkg/runtime/helper.go (L221-L243)
temporarily modifies the object that it is meant to encode. Callers of
client-go Create calls must be aware of that and pass in unique object if they
might get called concurrently.

It's not clear where these goroutines came from, but the data race seems genuine:

WARNING: DATA RACE
Read at 0x00c0001d66f0 by goroutine 70907:
  k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/apis/meta/v1.(*TypeMeta).GroupVersionKind()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/apis/meta/v1/meta.go:126 +0x64
  k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/runtime.WithVersionEncoder.Encode()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/runtime/helper.go:231 +0x176
  k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/runtime.(*WithVersionEncoder).Encode()
      <autogenerated>:1 +0xfb
  k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/runtime.Encode()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/runtime/codec.go:50 +0xb3
  k8s.io/kubernetes/vendor/k8s.io/client-go/rest.(*Request).Body()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/client-go/rest/request.go:469 +0x884
  k8s.io/kubernetes/vendor/k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta3.(*flowSchemas).Create()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta3/flowschema.go:118 +0x23c
  k8s.io/kubernetes/pkg/registry/flowcontrol/ensurer.(*flowSchemaWrapper).Create()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/pkg/registry/flowcontrol/ensurer/flowschema.go:156 +0x12b
  k8s.io/kubernetes/pkg/registry/flowcontrol/ensurer.ensureConfiguration()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/pkg/registry/flowcontrol/ensurer/strategy.go:235 +0x147
  k8s.io/kubernetes/pkg/registry/flowcontrol/ensurer.(*fsEnsurer).Ensure()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/pkg/registry/flowcontrol/ensurer/flowschema.go:121 +0xd2
  k8s.io/kubernetes/pkg/registry/flowcontrol/rest.ensureSuggestedConfiguration()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/pkg/registry/flowcontrol/rest/storage_flowcontrol.go:211 +0x417
  k8s.io/kubernetes/pkg/registry/flowcontrol/rest.ensure()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/pkg/registry/flowcontrol/rest/storage_flowcontrol.go:186 +0x99
  k8s.io/kubernetes/pkg/registry/flowcontrol/rest.(*bootstrapConfigurationEnsurer).ensureAPFBootstrapConfiguration.func1()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/pkg/registry/flowcontrol/rest/storage_flowcontrol.go:157 +0xb4
  k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:154 +0x7b
  k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/wait.poll()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/wait/poll.go:245 +0x57
  k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/wait.PollImmediateUntilWithContext()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/wait/poll.go:200 +0x59
  k8s.io/kubernetes/pkg/registry/flowcontrol/rest.(*bootstrapConfigurationEnsurer).ensureAPFBootstrapConfiguration()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/pkg/registry/flowcontrol/rest/storage_flowcontrol.go:153 +0x237
  k8s.io/kubernetes/pkg/registry/flowcontrol/rest.(*bootstrapConfigurationEnsurer).ensureAPFBootstrapConfiguration-fm()
      <autogenerated>:1 +0x58
  k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server.runPostStartHook.func1()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/hooks.go:199 +0xa1
  k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server.runPostStartHook()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/hooks.go:200 +0xda
  k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server.(*GenericAPIServer).RunPostStartHooks.func2()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/hooks.go:166 +0xb4

Previous write at 0x00c0001d66f0 by goroutine 69811:
  k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/apis/meta/v1.(*TypeMeta).SetGroupVersionKind()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/apis/meta/v1/meta.go:121 +0x193
  k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/runtime.WithVersionEncoder.Encode()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/runtime/helper.go:241 +0x3d9
  k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/runtime.(*WithVersionEncoder).Encode()
      <autogenerated>:1 +0xfb
  k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/runtime.Encode()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/runtime/codec.go:50 +0xb3
  k8s.io/kubernetes/vendor/k8s.io/client-go/rest.(*Request).Body()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/client-go/rest/request.go:469 +0x884
  k8s.io/kubernetes/vendor/k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta3.(*flowSchemas).Create()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta3/flowschema.go:118 +0x23c
  k8s.io/kubernetes/pkg/registry/flowcontrol/ensurer.(*flowSchemaWrapper).Create()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/pkg/registry/flowcontrol/ensurer/flowschema.go:156 +0x12b
  k8s.io/kubernetes/pkg/registry/flowcontrol/ensurer.ensureConfiguration()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/pkg/registry/flowcontrol/ensurer/strategy.go:235 +0x147
  k8s.io/kubernetes/pkg/registry/flowcontrol/ensurer.(*fsEnsurer).Ensure()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/pkg/registry/flowcontrol/ensurer/flowschema.go:121 +0xd2
  k8s.io/kubernetes/pkg/registry/flowcontrol/rest.ensureSuggestedConfiguration()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/pkg/registry/flowcontrol/rest/storage_flowcontrol.go:211 +0x417
  k8s.io/kubernetes/pkg/registry/flowcontrol/rest.ensure()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/pkg/registry/flowcontrol/rest/storage_flowcontrol.go:186 +0x99
  k8s.io/kubernetes/pkg/registry/flowcontrol/rest.(*bootstrapConfigurationEnsurer).ensureAPFBootstrapConfiguration.func1()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/pkg/registry/flowcontrol/rest/storage_flowcontrol.go:157 +0xb4
  k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:154 +0x7b
  k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/wait.poll()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/wait/poll.go:245 +0x57
  k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/wait.PollImmediateUntilWithContext()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/wait/poll.go:200 +0x59
  k8s.io/kubernetes/pkg/registry/flowcontrol/rest.(*bootstrapConfigurationEnsurer).ensureAPFBootstrapConfiguration()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/pkg/registry/flowcontrol/rest/storage_flowcontrol.go:153 +0x237
  k8s.io/kubernetes/pkg/registry/flowcontrol/rest.(*bootstrapConfigurationEnsurer).ensureAPFBootstrapConfiguration-fm()
      <autogenerated>:1 +0x58
  k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server.runPostStartHook.func1()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/hooks.go:199 +0xa1
  k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server.runPostStartHook()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/hooks.go:200 +0xda
  k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server.(*GenericAPIServer).RunPostStartHooks.func2()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/hooks.go:166 +0xb4

Goroutine 70907 (running) created at:
  k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server.(*GenericAPIServer).RunPostStartHooks()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/hooks.go:166 +0x167
  k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server.preparedGenericAPIServer.NonBlockingRun()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/genericapiserver.go:729 +0x21a
  k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server.preparedGenericAPIServer.Run()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/genericapiserver.go:578 +0x907
  k8s.io/kubernetes/vendor/k8s.io/kube-aggregator/pkg/apiserver.preparedAPIAggregator.Run()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go:447 +0xf8
  k8s.io/kubernetes/cmd/kube-apiserver/app/testing.StartTestServer.func3()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/cmd/kube-apiserver/app/testing/testserver.go:260 +0x109
  k8s.io/kubernetes/cmd/kube-apiserver/app/testing.StartTestServer.func9()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/cmd/kube-apiserver/app/testing/testserver.go:263 +0x47

Goroutine 69811 (running) created at:
  k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server.(*GenericAPIServer).RunPostStartHooks()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/hooks.go:166 +0x167
  k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server.preparedGenericAPIServer.NonBlockingRun()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/genericapiserver.go:729 +0x21a
  k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server.preparedGenericAPIServer.Run()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/genericapiserver.go:578 +0x907
  k8s.io/kubernetes/vendor/k8s.io/kube-aggregator/pkg/apiserver.preparedAPIAggregator.Run()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go:447 +0xf8
  k8s.io/kubernetes/cmd/kube-apiserver/app/testing.StartTestServer.func3()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/cmd/kube-apiserver/app/testing/testserver.go:260 +0x109
  k8s.io/kubernetes/cmd/kube-apiserver/app/testing.StartTestServer.func9()
      /home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/cmd/kube-apiserver/app/testing/testserver.go:263 +0x47
2023-04-05 15:59:22 +02:00

212 lines
6.8 KiB
Go

/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package ensurer
import (
"context"
"errors"
"fmt"
flowcontrolv1beta3 "k8s.io/api/flowcontrol/v1beta3"
"k8s.io/apimachinery/pkg/api/equality"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/sets"
flowcontrolclient "k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta3"
flowcontrollisters "k8s.io/client-go/listers/flowcontrol/v1beta3"
flowcontrolapisv1beta3 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta3"
)
var (
errObjectNotFlowSchema = errors.New("object is not a FlowSchema type")
)
// FlowSchemaEnsurer ensures the specified bootstrap configuration objects
type FlowSchemaEnsurer interface {
Ensure([]*flowcontrolv1beta3.FlowSchema) error
}
// FlowSchemaRemover is the interface that wraps the
// RemoveAutoUpdateEnabledObjects method.
//
// RemoveAutoUpdateEnabledObjects removes a set of bootstrap FlowSchema
// objects specified via their names. The function removes an object
// only if automatic update of the spec is enabled for it.
type FlowSchemaRemover interface {
RemoveAutoUpdateEnabledObjects([]string) error
}
// NewSuggestedFlowSchemaEnsurer returns a FlowSchemaEnsurer instance that
// can be used to ensure a set of suggested FlowSchema configuration objects.
func NewSuggestedFlowSchemaEnsurer(client flowcontrolclient.FlowSchemaInterface, lister flowcontrollisters.FlowSchemaLister) FlowSchemaEnsurer {
wrapper := &flowSchemaWrapper{
client: client,
lister: lister,
}
return &fsEnsurer{
strategy: newSuggestedEnsureStrategy(wrapper),
wrapper: wrapper,
}
}
// NewMandatoryFlowSchemaEnsurer returns a FlowSchemaEnsurer instance that
// can be used to ensure a set of mandatory FlowSchema configuration objects.
func NewMandatoryFlowSchemaEnsurer(client flowcontrolclient.FlowSchemaInterface, lister flowcontrollisters.FlowSchemaLister) FlowSchemaEnsurer {
wrapper := &flowSchemaWrapper{
client: client,
lister: lister,
}
return &fsEnsurer{
strategy: newMandatoryEnsureStrategy(wrapper),
wrapper: wrapper,
}
}
// NewFlowSchemaRemover returns a FlowSchemaRemover instance that
// can be used to remove a set of FlowSchema configuration objects.
func NewFlowSchemaRemover(client flowcontrolclient.FlowSchemaInterface, lister flowcontrollisters.FlowSchemaLister) FlowSchemaRemover {
return &fsEnsurer{
wrapper: &flowSchemaWrapper{
client: client,
lister: lister,
},
}
}
// GetFlowSchemaRemoveCandidates returns a list of FlowSchema object
// names that are candidates for deletion from the cluster.
// bootstrap: a set of hard coded FlowSchema configuration objects
// kube-apiserver maintains in-memory.
func GetFlowSchemaRemoveCandidates(lister flowcontrollisters.FlowSchemaLister, bootstrap []*flowcontrolv1beta3.FlowSchema) ([]string, error) {
fsList, err := lister.List(labels.Everything())
if err != nil {
return nil, fmt.Errorf("failed to list FlowSchema - %w", err)
}
bootstrapNames := sets.String{}
for i := range bootstrap {
bootstrapNames.Insert(bootstrap[i].GetName())
}
currentObjects := make([]metav1.Object, len(fsList))
for i := range fsList {
currentObjects[i] = fsList[i]
}
return getDanglingBootstrapObjectNames(bootstrapNames, currentObjects), nil
}
type fsEnsurer struct {
strategy ensureStrategy
wrapper configurationWrapper
}
func (e *fsEnsurer) Ensure(flowSchemas []*flowcontrolv1beta3.FlowSchema) error {
for _, flowSchema := range flowSchemas {
// This code gets called by different goroutines. To avoid race conditions when
// https://github.com/kubernetes/kubernetes/blob/330b5a2b8dbd681811cb8235947557c99dd8e593/staging/src/k8s.io/apimachinery/pkg/runtime/helper.go#L221-L243
// temporarily modifies the TypeMeta, we have to make a copy here.
if err := ensureConfiguration(e.wrapper, e.strategy, flowSchema.DeepCopy()); err != nil {
return err
}
}
return nil
}
func (e *fsEnsurer) RemoveAutoUpdateEnabledObjects(flowSchemas []string) error {
for _, flowSchema := range flowSchemas {
if err := removeAutoUpdateEnabledConfiguration(e.wrapper, flowSchema); err != nil {
return err
}
}
return nil
}
// flowSchemaWrapper abstracts all FlowSchema specific logic, with this
// we can manage all boiler plate code in one place.
type flowSchemaWrapper struct {
client flowcontrolclient.FlowSchemaInterface
lister flowcontrollisters.FlowSchemaLister
}
func (fs *flowSchemaWrapper) TypeName() string {
return "FlowSchema"
}
func (fs *flowSchemaWrapper) Create(object runtime.Object) (runtime.Object, error) {
fsObject, ok := object.(*flowcontrolv1beta3.FlowSchema)
if !ok {
return nil, errObjectNotFlowSchema
}
return fs.client.Create(context.TODO(), fsObject, metav1.CreateOptions{FieldManager: fieldManager})
}
func (fs *flowSchemaWrapper) Update(object runtime.Object) (runtime.Object, error) {
fsObject, ok := object.(*flowcontrolv1beta3.FlowSchema)
if !ok {
return nil, errObjectNotFlowSchema
}
return fs.client.Update(context.TODO(), fsObject, metav1.UpdateOptions{FieldManager: fieldManager})
}
func (fs *flowSchemaWrapper) Get(name string) (configurationObject, error) {
return fs.lister.Get(name)
}
func (fs *flowSchemaWrapper) Delete(name string) error {
return fs.client.Delete(context.TODO(), name, metav1.DeleteOptions{})
}
func (fs *flowSchemaWrapper) CopySpec(bootstrap, current runtime.Object) error {
bootstrapFS, ok := bootstrap.(*flowcontrolv1beta3.FlowSchema)
if !ok {
return errObjectNotFlowSchema
}
currentFS, ok := current.(*flowcontrolv1beta3.FlowSchema)
if !ok {
return errObjectNotFlowSchema
}
specCopy := bootstrapFS.Spec.DeepCopy()
currentFS.Spec = *specCopy
return nil
}
func (fs *flowSchemaWrapper) HasSpecChanged(bootstrap, current runtime.Object) (bool, error) {
bootstrapFS, ok := bootstrap.(*flowcontrolv1beta3.FlowSchema)
if !ok {
return false, errObjectNotFlowSchema
}
currentFS, ok := current.(*flowcontrolv1beta3.FlowSchema)
if !ok {
return false, errObjectNotFlowSchema
}
return flowSchemaSpecChanged(bootstrapFS, currentFS), nil
}
func flowSchemaSpecChanged(expected, actual *flowcontrolv1beta3.FlowSchema) bool {
copiedExpectedFlowSchema := expected.DeepCopy()
flowcontrolapisv1beta3.SetObjectDefaults_FlowSchema(copiedExpectedFlowSchema)
return !equality.Semantic.DeepEqual(copiedExpectedFlowSchema.Spec, actual.Spec)
}