github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #115554 from yt2985/cleanSA

Browse Source 
LegacyServiceAccountTokenCleanUp alpha
This commit is contained in:
Kubernetes Prow Robot
2023-05-26 08:54:53 -07:00
committed by GitHub
parent 1c9f08a1c5 133eff3df4
commit c35a2775b2
24 changed files with 1134 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

251
test/integration/serviceaccount/legacy_service_account_token_clean_up_test.go Normal file
View File

@@ -0,0 +1,251 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package serviceaccount
// This file tests the legacy service account token cleaning-up.
import (
"context"
"fmt"
"testing"
"time"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
utilfeature "k8s.io/apiserver/pkg/util/feature"
clientinformers "k8s.io/client-go/informers"
clientset "k8s.io/client-go/kubernetes"
listersv1 "k8s.io/client-go/listers/core/v1"
featuregatetesting "k8s.io/component-base/featuregate/testing"
serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount"
"k8s.io/kubernetes/pkg/controlplane/controller/legacytokentracking"
kubefeatures "k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/serviceaccount"
"k8s.io/utils/clock"
testingclock "k8s.io/utils/clock/testing"
)
const (
dateFormat = "2006-01-02"
cleanUpPeriod = 24 * time.Hour
syncInterval = 1 * time.Second
)
func TestLegacyServiceAccountTokenCleanUp(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, kubefeatures.LegacyServiceAccountTokenCleanUp, true)()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
fakeClock := testingclock.NewFakeClock(time.Now().UTC())
c, config, stopFunc, informers, err := startServiceAccountTestServerAndWaitForCaches(ctx, t)
defer stopFunc()
if err != nil {
t.Fatalf("failed to setup ServiceAccounts server: %v", err)
}
// start legacy service account token cleaner
startLegacyServiceAccountTokenCleaner(ctx, c, fakeClock, informers)
// wait configmap to label with tracking date
if err := wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) {
configMap, err := c.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(ctx, legacytokentracking.ConfigMapName, metav1.GetOptions{})
if err != nil {
return false, err
}
_, exist := configMap.Data[legacytokentracking.ConfigMapDataKey]
if !exist {
return false, fmt.Errorf("configMap does not have since label")
}
return true, nil
}); err != nil {
t.Fatalf("failed to wait configmap starts to track: %v", err)
}
// create service account
myns := "clean-ns"
_, err = c.CoreV1().Namespaces().Create(context.TODO(), &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: myns}}, metav1.CreateOptions{})
if err != nil && !apierrors.IsAlreadyExists(err) {
t.Fatalf("could not create namespace: %v", err)
}
mysa, err := c.CoreV1().ServiceAccounts(myns).Create(context.TODO(), &v1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: readOnlyServiceAccountName}}, metav1.CreateOptions{})
if err != nil {
t.Fatalf("Service Account not created: %v", err)
}
tests := []struct {
name string
secretName string
secretTokenData string
expectCleanedUp bool
lastUsedLabel bool
isPodMounted bool
isManual bool
}{
{
name: "auto created legacy token without pod binding",
secretName: "auto-token-without-pod-mounting-a",
lastUsedLabel: true,
isManual: false,
isPodMounted: false,
expectCleanedUp: true,
},
{
name: "manually created legacy token",
secretName: "manual-token",
lastUsedLabel: true,
isManual: true,
isPodMounted: false,
expectCleanedUp: false,
},
{
name: "auto created legacy token with pod binding",
secretName: "auto-token-with-pod-mounting",
lastUsedLabel: true,
isManual: false,
isPodMounted: true,
expectCleanedUp: false,
},
{
name: "auto created legacy token without pod binding, secret has not been used after tracking",
secretName: "auto-token-without-pod-mounting-b",
lastUsedLabel: false,
isManual: false,
isPodMounted: false,
expectCleanedUp: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
// create secret
secret, err := createServiceAccountToken(c, mysa, myns, test.secretName)
if err != nil {
t.Fatalf("Secret not created: %v", err)
}
if !test.isManual {
if err := addReferencedServiceAccountToken(c, myns, readOnlyServiceAccountName, secret); err != nil {
t.Fatal(err)
}
}
podLister := informers.Core().V1().Pods().Lister()
if test.isPodMounted {
_, err = createAutotokenMountedPod(c, myns, test.secretName, podLister)
if err != nil {
t.Fatalf("Pod not created: %v", err)
}
}
myConfig := *config
wh := &warningHandler{}
myConfig.WarningHandler = wh
myConfig.BearerToken = string(string(secret.Data[v1.ServiceAccountTokenKey]))
roClient := clientset.NewForConfigOrDie(&myConfig)
// the secret should not be labeled with LastUsedLabelKey.
liveSecret, err := c.CoreV1().Secrets(myns).Get(context.TODO(), test.secretName, metav1.GetOptions{})
if err != nil {
t.Fatalf("Could not get secret: %v", err)
}
_, ok := liveSecret.GetLabels()[serviceaccount.LastUsedLabelKey]
if ok {
t.Fatalf("Secret %s should not have the lastUsed label", test.secretName)
}
// authenticate legacy tokens
if test.lastUsedLabel {
doServiceAccountAPIRequests(t, roClient, myns, true, true, false)
// all service account tokens should be labeled with LastUsedLabelKey.
liveSecret, err = c.CoreV1().Secrets(myns).Get(context.TODO(), test.secretName, metav1.GetOptions{})
if err != nil {
t.Fatalf("Could not get secret: %v", err)
}
lastUsed, ok := liveSecret.GetLabels()[serviceaccount.LastUsedLabelKey]
if !ok {
t.Fatalf("The secret %s should be labeled lastUsed time: %s", test.secretName, lastUsed)
} else {
t.Logf("The secret %s has been labeled with %s", test.secretName, lastUsed)
}
}
_, err = c.CoreV1().Secrets(myns).Get(context.TODO(), test.secretName, metav1.GetOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
t.Fatalf("The secret %s should not be cleaned up, err: %v", test.secretName, err)
} else {
t.Fatalf("Failed to get secret %s, err: %v", test.secretName, err)
}
}
fakeClock.Step(cleanUpPeriod + 24*time.Hour)
time.Sleep(2 * syncInterval)
liveSecret, err = c.CoreV1().Secrets(myns).Get(context.TODO(), test.secretName, metav1.GetOptions{})
if test.expectCleanedUp && err == nil {
t.Fatalf("The secret %s should be cleaned up. time: %v; creationTime: %v", test.secretName, fakeClock.Now().UTC(), liveSecret.CreationTimestamp)
} else if !test.expectCleanedUp && err != nil {
if apierrors.IsNotFound(err) {
t.Fatalf("The secret %s should not be cleaned up, err: %v", test.secretName, err)
} else {
t.Fatalf("Failed to get secret %s, err: %v", test.secretName, err)
}
}
})
}
}
func startLegacyServiceAccountTokenCleaner(ctx context.Context, client clientset.Interface, fakeClock clock.Clock, informers clientinformers.SharedInformerFactory) {
legacySATokenCleaner, _ := serviceaccountcontroller.NewLegacySATokenCleaner(
informers.Core().V1().ServiceAccounts(),
informers.Core().V1().Secrets(),
informers.Core().V1().Pods(),
client,
fakeClock,
serviceaccountcontroller.LegacySATokenCleanerOptions{
SyncInterval: syncInterval,
CleanUpPeriod: cleanUpPeriod,
})
go legacySATokenCleaner.Run(ctx)
informers.Start(ctx.Done())
}
func createAutotokenMountedPod(c clientset.Interface, ns, secretName string, podLister listersv1.PodLister) (*v1.Pod, error) {
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "token-bound-pod",
Namespace: ns,
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{Name: "name", Image: "image"},
},
Volumes: []v1.Volume{{Name: "foo", VolumeSource: v1.VolumeSource{Secret: &v1.SecretVolumeSource{SecretName: secretName}}}},
},
}
pod, err := c.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{})
if err != nil {
return nil, fmt.Errorf("failed to create pod with token (%s:%s) bound, err: %v", ns, secretName, err)
}
err = wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) {
pod, err = podLister.Pods(ns).Get("token-bound-pod")
if err != nil {
return false, fmt.Errorf("failed to get pod with token (%s:%s) bound, err: %v", ns, secretName, err)
}
return true, nil
})
return pod, nil
}

20
test/integration/serviceaccount/service_account_test.go
View File

@@ -52,8 +52,6 @@ import (
const (
readOnlyServiceAccountName = "ro"
readWriteServiceAccountName = "rw"
dateFormat = "2006-01-02"
)
func TestServiceAccountAutoCreate(t *testing.T) {
@@ -61,7 +59,7 @@ func TestServiceAccountAutoCreate(t *testing.T) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
c, _, stopFunc, err := startServiceAccountTestServerAndWaitForCaches(ctx, t)
c, _, stopFunc, _, err := startServiceAccountTestServerAndWaitForCaches(ctx, t)
defer stopFunc()
if err != nil {
t.Fatalf("failed to setup ServiceAccounts server: %v", err)
@@ -102,7 +100,7 @@ func TestServiceAccountTokenAutoMount(t *testing.T) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
c, _, stopFunc, err := startServiceAccountTestServerAndWaitForCaches(ctx, t)
c, _, stopFunc, _, err := startServiceAccountTestServerAndWaitForCaches(ctx, t)
defer stopFunc()
if err != nil {
t.Fatalf("failed to setup ServiceAccounts server: %v", err)
@@ -148,7 +146,7 @@ func TestServiceAccountTokenAuthentication(t *testing.T) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
c, config, stopFunc, err := startServiceAccountTestServerAndWaitForCaches(ctx, t)
c, config, stopFunc, _, err := startServiceAccountTestServerAndWaitForCaches(ctx, t)
defer stopFunc()
if err != nil {
t.Fatalf("failed to setup ServiceAccounts server: %v", err)
@@ -229,7 +227,7 @@ func TestLegacyServiceAccountTokenTracking(t *testing.T) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
c, config, stopFunc, err := startServiceAccountTestServerAndWaitForCaches(ctx, t)
c, config, stopFunc, _, err := startServiceAccountTestServerAndWaitForCaches(ctx, t)
defer stopFunc()
if err != nil {
t.Fatalf("failed to setup ServiceAccounts server: %v", err)
@@ -347,7 +345,7 @@ func TestLegacyServiceAccountTokenTracking(t *testing.T) {
// startServiceAccountTestServerAndWaitForCaches returns a started server
// It is the responsibility of the caller to ensure the returned stopFunc is called
func startServiceAccountTestServerAndWaitForCaches(ctx context.Context, t *testing.T) (clientset.Interface, *restclient.Config, func(), error) {
func startServiceAccountTestServerAndWaitForCaches(ctx context.Context, t *testing.T) (clientset.Interface, *restclient.Config, func(), clientinformers.SharedInformerFactory, error) {
var serviceAccountKey interface{}
ctx, cancel := context.WithCancel(ctx)
@@ -404,7 +402,7 @@ func startServiceAccountTestServerAndWaitForCaches(ctx context.Context, t *testi
// Start the service account and service account token controllers
tokenGenerator, err := serviceaccount.JWTTokenGenerator(serviceaccount.LegacyIssuer, serviceAccountKey)
if err != nil {
return rootClientset, clientConfig, stop, err
return rootClientset, clientConfig, stop, informers, err
}
tokenController, err := serviceaccountcontroller.NewTokensController(
informers.Core().V1().ServiceAccounts(),
@@ -415,7 +413,7 @@ func startServiceAccountTestServerAndWaitForCaches(ctx context.Context, t *testi
},
)
if err != nil {
return rootClientset, clientConfig, stop, err
return rootClientset, clientConfig, stop, informers, err
}
go tokenController.Run(ctx, 1)
@@ -426,7 +424,7 @@ func startServiceAccountTestServerAndWaitForCaches(ctx context.Context, t *testi
serviceaccountcontroller.DefaultServiceAccountsControllerOptions(),
)
if err != nil {
return rootClientset, clientConfig, stop, err
return rootClientset, clientConfig, stop, informers, err
}
informers.Start(ctx.Done())
go serviceAccountController.Run(ctx, 5)
@@ -437,7 +435,7 @@ func startServiceAccountTestServerAndWaitForCaches(ctx context.Context, t *testi
// thus we wait until caches have synced
informers.WaitForCacheSync(ctx.Done())
return rootClientset, clientConfig, stop, nil
return rootClientset, clientConfig, stop, informers, nil
}
func getServiceAccount(c clientset.Interface, ns string, name string, shouldWait bool) (*v1.ServiceAccount, error) {