github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

encryptionconfig/controller: run unit tests faster

Browse Source 
Signed-off-by: Monis Khan <mok@microsoft.com>
This commit is contained in:
Monis Khan
2023-10-31 11:59:37 -04:00
parent 74fefd877f
commit 6ac7da1da8
2 changed files with 27 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller.go
View File

@@ -38,8 +38,8 @@ const workqueueKey = "key"
// EncryptionConfigFileChangePollDuration is exposed so that integration tests can crank up the reload speed. // EncryptionConfigFileChangePollDuration is exposed so that integration tests can crank up the reload speed.
var EncryptionConfigFileChangePollDuration = time.Minute var EncryptionConfigFileChangePollDuration = time.Minute
// DynamicKMSEncryptionConfigContent which can dynamically handle changes in encryption config file. // DynamicEncryptionConfigContent which can dynamically handle changes in encryption config file.
type DynamicKMSEncryptionConfigContent struct { type DynamicEncryptionConfigContent struct {
name string name string
// filePath is the path of the file to read. // filePath is the path of the file to read.
@@ -72,8 +72,8 @@ func NewDynamicEncryptionConfiguration(
dynamicTransformers *encryptionconfig.DynamicTransformers, dynamicTransformers *encryptionconfig.DynamicTransformers,
configContentHash string, configContentHash string,
apiServerID string, apiServerID string,
) *DynamicKMSEncryptionConfigContent { ) *DynamicEncryptionConfigContent {
encryptionConfig := &DynamicKMSEncryptionConfigContent{ return &DynamicEncryptionConfigContent{
name: name, name: name,
filePath: filePath, filePath: filePath,
lastLoadedEncryptionConfigHash: configContentHash, lastLoadedEncryptionConfigHash: configContentHash,
@@ -85,13 +85,10 @@ func NewDynamicEncryptionConfiguration(
}, },
loadEncryptionConfig: encryptionconfig.LoadEncryptionConfig, loadEncryptionConfig: encryptionconfig.LoadEncryptionConfig,
} }
encryptionConfig.queue.Add(workqueueKey) // to avoid missing any file changes that occur in between the initial load and Run
return encryptionConfig
} }
// Run starts the controller and blocks until ctx is canceled. // Run starts the controller and blocks until ctx is canceled.
func (d *DynamicKMSEncryptionConfigContent) Run(ctx context.Context) { func (d *DynamicEncryptionConfigContent) Run(ctx context.Context) {
defer utilruntime.HandleCrash() defer utilruntime.HandleCrash()
klog.InfoS("Starting controller", "name", d.name) klog.InfoS("Starting controller", "name", d.name)
@@ -134,13 +131,13 @@ func (d *DynamicKMSEncryptionConfigContent) Run(ctx context.Context) {
} }
// runWorker to process file content // runWorker to process file content
func (d *DynamicKMSEncryptionConfigContent) runWorker(ctx context.Context) { func (d *DynamicEncryptionConfigContent) runWorker(ctx context.Context) {
for d.processNextWorkItem(ctx) { for d.processNextWorkItem(ctx) {
} }
} }
// processNextWorkItem processes file content when there is a message in the queue. // processNextWorkItem processes file content when there is a message in the queue.
func (d *DynamicKMSEncryptionConfigContent) processNextWorkItem(serverCtx context.Context) bool { func (d *DynamicEncryptionConfigContent) processNextWorkItem(serverCtx context.Context) bool {
// key here is dummy item in the queue to trigger file content processing. // key here is dummy item in the queue to trigger file content processing.
key, quit := d.queue.Get() key, quit := d.queue.Get()
if quit { if quit {
@@ -153,7 +150,7 @@ func (d *DynamicKMSEncryptionConfigContent) processNextWorkItem(serverCtx contex
return true return true
} }
func (d *DynamicKMSEncryptionConfigContent) processWorkItem(serverCtx context.Context, workqueueKey interface{}) { func (d *DynamicEncryptionConfigContent) processWorkItem(serverCtx context.Context, workqueueKey interface{}) {
var ( var (
updatedEffectiveConfig bool updatedEffectiveConfig bool
err error err error
@@ -219,7 +216,7 @@ func (d *DynamicKMSEncryptionConfigContent) processWorkItem(serverCtx context.Co
} }
// loadEncryptionConfig processes the next set of content from the file. // loadEncryptionConfig processes the next set of content from the file.
func (d *DynamicKMSEncryptionConfigContent) processEncryptionConfig(ctx context.Context) ( func (d *DynamicEncryptionConfigContent) processEncryptionConfig(ctx context.Context) (
_ *encryptionconfig.EncryptionConfiguration, _ *encryptionconfig.EncryptionConfiguration,
configChanged bool, configChanged bool,
_ error, _ error,
@@ -250,7 +247,10 @@ func (d *DynamicKMSEncryptionConfigContent) processEncryptionConfig(ctx context.
return encryptionConfiguration, true, nil return encryptionConfiguration, true, nil
} }
func (d *DynamicKMSEncryptionConfigContent) validateNewTransformersHealth( // minKMSPluginCloseGracePeriod can be lowered in unit tests to make the health check poll faster
var minKMSPluginCloseGracePeriod = 10 * time.Second
func (d *DynamicEncryptionConfigContent) validateNewTransformersHealth(
ctx context.Context, ctx context.Context,
kmsPluginHealthzCheck healthz.HealthChecker, kmsPluginHealthzCheck healthz.HealthChecker,
kmsPluginCloseGracePeriod time.Duration, kmsPluginCloseGracePeriod time.Duration,
@@ -258,8 +258,8 @@ func (d *DynamicKMSEncryptionConfigContent) validateNewTransformersHealth(
// test if new transformers are healthy // test if new transformers are healthy
var healthCheckError error var healthCheckError error
if kmsPluginCloseGracePeriod < 10*time.Second { if kmsPluginCloseGracePeriod < minKMSPluginCloseGracePeriod {
kmsPluginCloseGracePeriod = 10 * time.Second kmsPluginCloseGracePeriod = minKMSPluginCloseGracePeriod
} }
// really make sure that the immediate check does not hang // really make sure that the immediate check does not hang

18
staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller_test.go
View File

@@ -37,6 +37,10 @@ import (
) )
func TestController(t *testing.T) { func TestController(t *testing.T) {
origMinKMSPluginCloseGracePeriod := minKMSPluginCloseGracePeriod
t.Cleanup(func() { minKMSPluginCloseGracePeriod = origMinKMSPluginCloseGracePeriod })
minKMSPluginCloseGracePeriod = 300 * time.Millisecond
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)()
const expectedSuccessMetricValue = ` const expectedSuccessMetricValue = `
@@ -141,7 +145,7 @@ apiserver_encryption_config_controller_automatic_reload_failures_total{apiserver
err: fmt.Errorf("mockingly failing"), err: fmt.Errorf("mockingly failing"),
}, },
}, },
KMSCloseGracePeriod: time.Second, KMSCloseGracePeriod: 0, // use minKMSPluginCloseGracePeriod
EncryptionFileContentHash: "anything different", EncryptionFileContentHash: "anything different",
}, nil }, nil
}, },
@@ -255,14 +259,16 @@ apiserver_encryption_config_controller_automatic_reload_failures_total{apiserver
for _, test := range tests { for _, test := range tests {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
serverCtx, closeServer := context.WithCancel(context.Background()) ctxServer, closeServer := context.WithCancel(context.Background())
ctxTransformers, closeTransformers := context.WithCancel(ctxServer)
t.Cleanup(closeServer) t.Cleanup(closeServer)
t.Cleanup(closeTransformers)
legacyregistry.Reset() legacyregistry.Reset()
// load initial encryption config // load initial encryption config
encryptionConfiguration, err := encryptionconfig.LoadEncryptionConfig( encryptionConfiguration, err := encryptionconfig.LoadEncryptionConfig(
serverCtx, ctxTransformers,
"testdata/ec_config.yaml", "testdata/ec_config.yaml",
true, true,
"test-apiserver", "test-apiserver",
@@ -277,8 +283,8 @@ apiserver_encryption_config_controller_automatic_reload_failures_total{apiserver
encryptionconfig.NewDynamicTransformers( encryptionconfig.NewDynamicTransformers(
encryptionConfiguration.Transformers, encryptionConfiguration.Transformers,
encryptionConfiguration.HealthChecks[0], encryptionConfiguration.HealthChecks[0],
closeServer, closeTransformers,
encryptionConfiguration.KMSCloseGracePeriod, 0, // set grace period to 0 so that the time.Sleep in DynamicTransformers.Set finishes quickly
), ),
encryptionConfiguration.EncryptionFileContentHash, encryptionConfiguration.EncryptionFileContentHash,
"test-apiserver", "test-apiserver",
@@ -303,7 +309,7 @@ apiserver_encryption_config_controller_automatic_reload_failures_total{apiserver
return test.mockGetEncryptionConfigHash(ctx, filepath) return test.mockGetEncryptionConfigHash(ctx, filepath)
} }
d.Run(serverCtx) // this should block and run exactly one iteration of the worker loop d.Run(ctxServer) // this should block and run exactly one iteration of the worker loop
if test.wantECFileHash != d.lastLoadedEncryptionConfigHash { if test.wantECFileHash != d.lastLoadedEncryptionConfigHash {
t.Errorf("expected encryption config hash %q but got %q", test.wantECFileHash, d.lastLoadedEncryptionConfigHash) t.Errorf("expected encryption config hash %q but got %q", test.wantECFileHash, d.lastLoadedEncryptionConfigHash)