diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index b689c5f52e9..a98e3fc65f7 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -411,6 +411,12 @@ const ( // // Implement support for limiting pids in nodes SupportNodePidsLimit utilfeature.Feature = "SupportNodePidsLimit" + + // owner: @wk8 + // alpha: v1.14 + // + // Enables GMSA support for Windows workloads. + WindowsGMSA utilfeature.Feature = "WindowsGMSA" ) func init() { diff --git a/pkg/kubelet/dockershim/BUILD b/pkg/kubelet/dockershim/BUILD index 5915ed02124..c3f65639942 100644 --- a/pkg/kubelet/dockershim/BUILD +++ b/pkg/kubelet/dockershim/BUILD @@ -7,6 +7,8 @@ go_library( "doc.go", "docker_checkpoint.go", "docker_container.go", + "docker_container_unsupported.go", + "docker_container_windows.go", "docker_image.go", "docker_image_linux.go", "docker_image_unsupported.go", @@ -73,6 +75,7 @@ go_library( "@io_bazel_rules_go//go/platform:windows": [ "//pkg/kubelet/apis:go_default_library", "//pkg/kubelet/winstats:go_default_library", + "//vendor/golang.org/x/sys/windows/registry:go_default_library", ], "//conditions:default": [], }), @@ -84,6 +87,7 @@ go_test( "convert_test.go", "docker_checkpoint_test.go", "docker_container_test.go", + "docker_container_windows_test.go", "docker_image_test.go", "docker_sandbox_test.go", "docker_service_test.go", @@ -118,6 +122,9 @@ go_test( "@io_bazel_rules_go//go/platform:linux": [ "//staging/src/k8s.io/api/core/v1:go_default_library", ], + "@io_bazel_rules_go//go/platform:windows": [ + "//vendor/golang.org/x/sys/windows/registry:go_default_library", + ], "//conditions:default": [], }), ) diff --git a/pkg/kubelet/dockershim/docker_container.go b/pkg/kubelet/dockershim/docker_container.go index 3c6b9b48497..51156483f2c 100644 --- a/pkg/kubelet/dockershim/docker_container.go +++ b/pkg/kubelet/dockershim/docker_container.go @@ -114,8 +114,9 @@ func (ds *dockerService) CreateContainer(_ context.Context, r *runtimeapi.Create if iSpec := config.GetImage(); iSpec != nil { image = iSpec.Image } + containerName := makeContainerName(sandboxConfig, config) createConfig := dockertypes.ContainerCreateConfig{ - Name: makeContainerName(sandboxConfig, config), + Name: containerName, Config: &dockercontainer.Config{ // TODO: set User. Entrypoint: dockerstrslice.StrSlice(config.Command), @@ -162,15 +163,25 @@ func (ds *dockerService) CreateContainer(_ context.Context, r *runtimeapi.Create hc.SecurityOpt = append(hc.SecurityOpt, securityOpts...) - createResp, err := ds.client.CreateContainer(createConfig) + cleanupInfo, err := ds.applyPlatformSpecificDockerConfig(r, &createConfig) if err != nil { - createResp, err = recoverFromCreationConflictIfNeeded(ds.client, createConfig, err) + return nil, err + } + defer func() { + for _, err := range ds.performPlatformSpecificContainerCreationCleanup(cleanupInfo) { + klog.Warningf("error when cleaning up after container %v's creation: %v", containerName, err) + } + }() + + createResp, createErr := ds.client.CreateContainer(createConfig) + if createErr != nil { + createResp, createErr = recoverFromCreationConflictIfNeeded(ds.client, createConfig, createErr) } if createResp != nil { return &runtimeapi.CreateContainerResponse{ContainerId: createResp.ID}, nil } - return nil, err + return nil, createErr } // getContainerLogPath returns the container log path specified by kubelet and the real diff --git a/pkg/kubelet/dockershim/docker_container_unsupported.go b/pkg/kubelet/dockershim/docker_container_unsupported.go new file mode 100644 index 00000000000..10dea828f14 --- /dev/null +++ b/pkg/kubelet/dockershim/docker_container_unsupported.go @@ -0,0 +1,48 @@ +// +build !windows + +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dockershim + +import ( + dockertypes "github.com/docker/docker/api/types" + runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" +) + +type containerCreationCleanupInfo struct{} + +// applyPlatformSpecificDockerConfig applies platform-specific configurations to a dockertypes.ContainerCreateConfig struct. +// The containerCreationCleanupInfo struct it returns will be passed as is to performPlatformSpecificContainerCreationCleanup +// after the container has been created. +func (ds *dockerService) applyPlatformSpecificDockerConfig(*runtimeapi.CreateContainerRequest, *dockertypes.ContainerCreateConfig) (*containerCreationCleanupInfo, error) { + return nil, nil +} + +// performPlatformSpecificContainerCreationCleanup is responsible for doing any platform-specific cleanup +// after a container creation. Any errors it returns are simply logged, but do not fail the container +// creation. +func (ds *dockerService) performPlatformSpecificContainerCreationCleanup(cleanupInfo *containerCreationCleanupInfo) (errors []error) { + return +} + +// platformSpecificContainerCreationInitCleanup is called when dockershim +// is starting, and is meant to clean up any cruft left by previous runs +// creating containers. +// Errors are simply logged, but don't prevent dockershim from starting. +func (ds *dockerService) platformSpecificContainerCreationInitCleanup() (errors []error) { + return +} diff --git a/pkg/kubelet/dockershim/docker_container_windows.go b/pkg/kubelet/dockershim/docker_container_windows.go new file mode 100644 index 00000000000..768f0c1d499 --- /dev/null +++ b/pkg/kubelet/dockershim/docker_container_windows.go @@ -0,0 +1,216 @@ +// +build windows + +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dockershim + +import ( + "crypto/rand" + "encoding/hex" + "fmt" + "regexp" + + "golang.org/x/sys/windows/registry" + + dockertypes "github.com/docker/docker/api/types" + dockercontainer "github.com/docker/docker/api/types/container" + + runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" + "k8s.io/kubernetes/pkg/kubelet/kuberuntime" +) + +type containerCreationCleanupInfo struct { + gMSARegistryValueName string +} + +// applyPlatformSpecificDockerConfig applies platform-specific configurations to a dockertypes.ContainerCreateConfig struct. +// The containerCreationCleanupInfo struct it returns will be passed as is to performPlatformSpecificContainerCreationCleanup +// after the container has been created. +func (ds *dockerService) applyPlatformSpecificDockerConfig(request *runtimeapi.CreateContainerRequest, createConfig *dockertypes.ContainerCreateConfig) (*containerCreationCleanupInfo, error) { + cleanupInfo := &containerCreationCleanupInfo{} + + if err := applyGMSAConfig(request.GetConfig(), createConfig, cleanupInfo); err != nil { + return nil, err + } + + return cleanupInfo, nil +} + +// applyGMSAConfig looks at the kuberuntime.GMSASpecContainerAnnotationKey container annotation; if present, +// it copies its contents to a unique registry value, and sets a SecurityOpt on the config pointing to that registry value. +// We use registry values instead of files since their location cannot change - as opposed to credential spec files, +// whose location could potentially change down the line, or even be unknown (eg if docker is not installed on the +// C: drive) +// When docker supports passing a credential spec's contents directly, we should switch to using that +// as it will avoid cluttering the registry - there is a moby PR out for this: +// https://github.com/moby/moby/pull/38777 +func applyGMSAConfig(config *runtimeapi.ContainerConfig, createConfig *dockertypes.ContainerCreateConfig, cleanupInfo *containerCreationCleanupInfo) error { + credSpec := config.Annotations[kuberuntime.GMSASpecContainerAnnotationKey] + if credSpec == "" { + return nil + } + + valueName, err := copyGMSACredSpecToRegistryValue(credSpec) + if err != nil { + return err + } + + if createConfig.HostConfig == nil { + createConfig.HostConfig = &dockercontainer.HostConfig{} + } + + createConfig.HostConfig.SecurityOpt = append(createConfig.HostConfig.SecurityOpt, "credentialspec=registry://"+valueName) + cleanupInfo.gMSARegistryValueName = valueName + + return nil +} + +const ( + // same as https://github.com/moby/moby/blob/93d994e29c9cc8d81f1b0477e28d705fa7e2cd72/daemon/oci_windows.go#L23 + credentialSpecRegistryLocation = `SOFTWARE\Microsoft\Windows NT\CurrentVersion\Virtualization\Containers\CredentialSpecs` + // the prefix for the registry values we write GMSA cred specs to + gMSARegistryValueNamePrefix = "k8s-cred-spec-" + // the number of random bytes to generate suffixes for registry value names + gMSARegistryValueNameSuffixRandomBytes = 40 +) + +// registryKey is an interface wrapper around `registry.Key`, +// listing only the methods we care about here. +// It's mainly useful to easily allow mocking the registry in tests. +type registryKey interface { + SetStringValue(name, value string) error + DeleteValue(name string) error + ReadValueNames(n int) ([]string, error) + Close() error +} + +var registryCreateKeyFunc = func(baseKey registry.Key, path string, access uint32) (registryKey, bool, error) { + return registry.CreateKey(baseKey, path, access) +} + +// randomReader is only meant to ever be overridden for testing purposes, +// same idea as for `registryKey` above +var randomReader = rand.Reader + +// gMSARegistryValueNamesRegex is the regex used to detect gMSA cred spec +// registry values in `removeAllGMSARegistryValues` below. +var gMSARegistryValueNamesRegex = regexp.MustCompile(fmt.Sprintf("^%s[0-9a-f]{%d}$", gMSARegistryValueNamePrefix, 2*gMSARegistryValueNameSuffixRandomBytes)) + +// copyGMSACredSpecToRegistryKey copies the credential specs to a unique registry value, and returns its name. +func copyGMSACredSpecToRegistryValue(credSpec string) (string, error) { + valueName, err := gMSARegistryValueName() + if err != nil { + return "", err + } + + // write to the registry + key, _, err := registryCreateKeyFunc(registry.LOCAL_MACHINE, credentialSpecRegistryLocation, registry.SET_VALUE) + if err != nil { + return "", fmt.Errorf("unable to open registry key %q: %v", credentialSpecRegistryLocation, err) + } + defer key.Close() + if err = key.SetStringValue(valueName, credSpec); err != nil { + return "", fmt.Errorf("unable to write into registry value %q/%q: %v", credentialSpecRegistryLocation, valueName, err) + } + + return valueName, nil +} + +// gMSARegistryValueName computes the name of the registry value where to store the GMSA cred spec contents. +// The value's name is a purely random suffix appended to `gMSARegistryValueNamePrefix`. +func gMSARegistryValueName() (string, error) { + randomSuffix, err := randomString(gMSARegistryValueNameSuffixRandomBytes) + + if err != nil { + return "", fmt.Errorf("error when generating gMSA registry value name: %v", err) + } + + return gMSARegistryValueNamePrefix + randomSuffix, nil +} + +// randomString returns a random hex string. +func randomString(length int) (string, error) { + randBytes := make([]byte, length) + + if n, err := randomReader.Read(randBytes); err != nil || n != length { + if err == nil { + err = fmt.Errorf("only got %v random bytes, expected %v", n, length) + } + return "", fmt.Errorf("unable to generate random string: %v", err) + } + + return hex.EncodeToString(randBytes), nil +} + +// performPlatformSpecificContainerCreationCleanup is responsible for doing any platform-specific cleanup +// after a container creation. Any errors it returns are simply logged, but do not fail the container +// creation. +func (ds *dockerService) performPlatformSpecificContainerCreationCleanup(cleanupInfo *containerCreationCleanupInfo) (errors []error) { + if err := removeGMSARegistryValue(cleanupInfo); err != nil { + errors = append(errors, err) + } + + return +} + +func removeGMSARegistryValue(cleanupInfo *containerCreationCleanupInfo) error { + if cleanupInfo == nil || cleanupInfo.gMSARegistryValueName == "" { + return nil + } + + key, _, err := registryCreateKeyFunc(registry.LOCAL_MACHINE, credentialSpecRegistryLocation, registry.SET_VALUE) + if err != nil { + return fmt.Errorf("unable to open registry key %q: %v", credentialSpecRegistryLocation, err) + } + defer key.Close() + if err = key.DeleteValue(cleanupInfo.gMSARegistryValueName); err != nil { + return fmt.Errorf("unable to remove registry value %q/%q: %v", credentialSpecRegistryLocation, cleanupInfo.gMSARegistryValueName, err) + } + + return nil +} + +// platformSpecificContainerCreationInitCleanup is called when dockershim +// is starting, and is meant to clean up any cruft left by previous runs +// creating containers. +// Errors are simply logged, but don't prevent dockershim from starting. +func (ds *dockerService) platformSpecificContainerCreationInitCleanup() (errors []error) { + return removeAllGMSARegistryValues() +} + +func removeAllGMSARegistryValues() (errors []error) { + key, _, err := registryCreateKeyFunc(registry.LOCAL_MACHINE, credentialSpecRegistryLocation, registry.SET_VALUE) + if err != nil { + return []error{fmt.Errorf("unable to open registry key %q: %v", credentialSpecRegistryLocation, err)} + } + defer key.Close() + + valueNames, err := key.ReadValueNames(0) + if err != nil { + return []error{fmt.Errorf("unable to list values under registry key %q: %v", credentialSpecRegistryLocation, err)} + } + + for _, valueName := range valueNames { + if gMSARegistryValueNamesRegex.MatchString(valueName) { + if err = key.DeleteValue(valueName); err != nil { + errors = append(errors, fmt.Errorf("unable to remove registry value %q/%q: %v", credentialSpecRegistryLocation, valueName, err)) + } + } + } + + return +} diff --git a/pkg/kubelet/dockershim/docker_container_windows_test.go b/pkg/kubelet/dockershim/docker_container_windows_test.go new file mode 100644 index 00000000000..21441509df3 --- /dev/null +++ b/pkg/kubelet/dockershim/docker_container_windows_test.go @@ -0,0 +1,304 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dockershim + +import ( + "bytes" + "fmt" + "regexp" + "testing" + + dockertypes "github.com/docker/docker/api/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/windows/registry" + runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" +) + +type dummyRegistryKey struct { + setStringValueError error + setStringValueArgs [][]string + + deleteValueFunc func(name string) error + deleteValueArgs []string + + readValueNamesError error + readValueNamesReturn []string + readValueNamesArgs []int + + closed bool +} + +func (k *dummyRegistryKey) SetStringValue(name, value string) error { + k.setStringValueArgs = append(k.setStringValueArgs, []string{name, value}) + return k.setStringValueError +} + +func (k *dummyRegistryKey) DeleteValue(name string) error { + k.deleteValueArgs = append(k.deleteValueArgs, name) + if k.deleteValueFunc == nil { + return nil + } + return k.deleteValueFunc(name) +} + +func (k *dummyRegistryKey) ReadValueNames(n int) ([]string, error) { + k.readValueNamesArgs = append(k.readValueNamesArgs, n) + return k.readValueNamesReturn, k.readValueNamesError +} + +func (k *dummyRegistryKey) Close() error { + k.closed = true + return nil +} + +func TestApplyGMSAConfig(t *testing.T) { + dummyCredSpec := "test cred spec contents" + randomBytes := []byte{0x19, 0x0, 0x25, 0x45, 0x18, 0x52, 0x9e, 0x2a, 0x3d, 0xed, 0xb8, 0x5c, 0xde, 0xc0, 0x3c, 0xe2, 0x70, 0x55, 0x96, 0x47, 0x45, 0x9a, 0xb5, 0x31, 0xf0, 0x7a, 0xf5, 0xeb, 0x1c, 0x54, 0x95, 0xfd, 0xa7, 0x9, 0x43, 0x5c, 0xe8, 0x2a, 0xb8, 0x9c} + expectedHex := "1900254518529e2a3dedb85cdec03ce270559647459ab531f07af5eb1c5495fda709435ce82ab89c" + expectedValueName := "k8s-cred-spec-" + expectedHex + + containerConfigWithGMSAAnnotation := &runtimeapi.ContainerConfig{ + Annotations: map[string]string{"container.alpha.windows.kubernetes.io/gmsa-credential-spec": dummyCredSpec}, + } + + t.Run("happy path", func(t *testing.T) { + key := &dummyRegistryKey{} + defer setRegistryCreateKeyFunc(t, key)() + defer setRandomReader(randomBytes)() + + createConfig := &dockertypes.ContainerCreateConfig{} + cleanupInfo := &containerCreationCleanupInfo{} + err := applyGMSAConfig(containerConfigWithGMSAAnnotation, createConfig, cleanupInfo) + + assert.Nil(t, err) + + // the registry key should have been properly created + if assert.Equal(t, 1, len(key.setStringValueArgs)) { + assert.Equal(t, []string{expectedValueName, dummyCredSpec}, key.setStringValueArgs[0]) + } + assert.True(t, key.closed) + + // the create config's security opt should have been populated + if assert.NotNil(t, createConfig.HostConfig) { + assert.Equal(t, createConfig.HostConfig.SecurityOpt, []string{"credentialspec=registry://" + expectedValueName}) + } + + // and the name of that value should have been saved to the cleanup info + assert.Equal(t, expectedValueName, cleanupInfo.gMSARegistryValueName) + }) + t.Run("happy path with a truly random string", func(t *testing.T) { + defer setRegistryCreateKeyFunc(t, &dummyRegistryKey{})() + + createConfig := &dockertypes.ContainerCreateConfig{} + cleanupInfo := &containerCreationCleanupInfo{} + err := applyGMSAConfig(containerConfigWithGMSAAnnotation, createConfig, cleanupInfo) + + assert.Nil(t, err) + + if assert.NotNil(t, createConfig.HostConfig) && assert.Equal(t, 1, len(createConfig.HostConfig.SecurityOpt)) { + secOpt := createConfig.HostConfig.SecurityOpt[0] + + expectedPrefix := "credentialspec=registry://k8s-cred-spec-" + assert.Equal(t, expectedPrefix, secOpt[:len(expectedPrefix)]) + + hex := secOpt[len(expectedPrefix):] + hexRegex := regexp.MustCompile("^[0-9a-f]{80}$") + assert.True(t, hexRegex.MatchString(hex)) + assert.NotEqual(t, expectedHex, hex) + + assert.Equal(t, "k8s-cred-spec-"+hex, cleanupInfo.gMSARegistryValueName) + } + }) + t.Run("when there's an error generating the random value name", func(t *testing.T) { + defer setRandomReader([]byte{})() + + err := applyGMSAConfig(containerConfigWithGMSAAnnotation, &dockertypes.ContainerCreateConfig{}, &containerCreationCleanupInfo{}) + + require.NotNil(t, err) + assert.Contains(t, err.Error(), "error when generating gMSA registry value name: unable to generate random string") + }) + t.Run("if there's an error opening the registry key", func(t *testing.T) { + defer setRegistryCreateKeyFunc(t, &dummyRegistryKey{}, fmt.Errorf("dummy error"))() + + err := applyGMSAConfig(containerConfigWithGMSAAnnotation, &dockertypes.ContainerCreateConfig{}, &containerCreationCleanupInfo{}) + + require.NotNil(t, err) + assert.Contains(t, err.Error(), "unable to open registry key") + }) + t.Run("if there's an error writing to the registry key", func(t *testing.T) { + key := &dummyRegistryKey{} + key.setStringValueError = fmt.Errorf("dummy error") + defer setRegistryCreateKeyFunc(t, key)() + + err := applyGMSAConfig(containerConfigWithGMSAAnnotation, &dockertypes.ContainerCreateConfig{}, &containerCreationCleanupInfo{}) + + if assert.NotNil(t, err) { + assert.Contains(t, err.Error(), "unable to write into registry value") + } + assert.True(t, key.closed) + }) + t.Run("if there is no GMSA annotation", func(t *testing.T) { + createConfig := &dockertypes.ContainerCreateConfig{} + + err := applyGMSAConfig(&runtimeapi.ContainerConfig{}, createConfig, &containerCreationCleanupInfo{}) + + assert.Nil(t, err) + assert.Nil(t, createConfig.HostConfig) + }) +} + +func TestRemoveGMSARegistryValue(t *testing.T) { + valueName := "k8s-cred-spec-1900254518529e2a3dedb85cdec03ce270559647459ab531f07af5eb1c5495fda709435ce82ab89c" + cleanupInfoWithValue := &containerCreationCleanupInfo{gMSARegistryValueName: valueName} + + t.Run("it does remove the registry value", func(t *testing.T) { + key := &dummyRegistryKey{} + defer setRegistryCreateKeyFunc(t, key)() + + err := removeGMSARegistryValue(cleanupInfoWithValue) + + assert.Nil(t, err) + + // the registry key should have been properly deleted + if assert.Equal(t, 1, len(key.deleteValueArgs)) { + assert.Equal(t, []string{valueName}, key.deleteValueArgs) + } + assert.True(t, key.closed) + }) + t.Run("if there's an error opening the registry key", func(t *testing.T) { + defer setRegistryCreateKeyFunc(t, &dummyRegistryKey{}, fmt.Errorf("dummy error"))() + + err := removeGMSARegistryValue(cleanupInfoWithValue) + + require.NotNil(t, err) + assert.Contains(t, err.Error(), "unable to open registry key") + }) + t.Run("if there's an error deleting from the registry key", func(t *testing.T) { + key := &dummyRegistryKey{} + key.deleteValueFunc = func(name string) error { return fmt.Errorf("dummy error") } + defer setRegistryCreateKeyFunc(t, key)() + + err := removeGMSARegistryValue(cleanupInfoWithValue) + + if assert.NotNil(t, err) { + assert.Contains(t, err.Error(), "unable to remove registry value") + } + assert.True(t, key.closed) + }) + t.Run("if there's no registry value to be removed, it does nothing", func(t *testing.T) { + key := &dummyRegistryKey{} + defer setRegistryCreateKeyFunc(t, key)() + + err := removeGMSARegistryValue(&containerCreationCleanupInfo{}) + + assert.Nil(t, err) + assert.Equal(t, 0, len(key.deleteValueArgs)) + }) +} + +func TestRemoveAllGMSARegistryValues(t *testing.T) { + cred1 := "k8s-cred-spec-1900254518529e2a3dedb85cdec03ce270559647459ab531f07af5eb1c5495fda709435ce82ab89c" + cred2 := "k8s-cred-spec-8891436007c795a904fdf77b5348e94305e4c48c5f01c47e7f65e980dc7edda85f112715891d65fd" + cred3 := "k8s-cred-spec-2f11f1c9e4f8182fe13caa708bd42b2098c8eefc489d6cc98806c058ccbe4cb3703b9ade61ce59a1" + cred4 := "k8s-cred-spec-dc532f189598a8220a1e538f79081eee979f94fbdbf8d37e36959485dee57157c03742d691e1fae2" + + t.Run("it removes the keys matching the k8s creds pattern", func(t *testing.T) { + key := &dummyRegistryKey{readValueNamesReturn: []string{cred1, "other_creds", cred2}} + defer setRegistryCreateKeyFunc(t, key)() + + errors := removeAllGMSARegistryValues() + + assert.Equal(t, 0, len(errors)) + assert.Equal(t, []string{cred1, cred2}, key.deleteValueArgs) + assert.Equal(t, []int{0}, key.readValueNamesArgs) + assert.True(t, key.closed) + }) + t.Run("it ignores errors and does a best effort at removing all k8s creds", func(t *testing.T) { + key := &dummyRegistryKey{ + readValueNamesReturn: []string{cred1, cred2, cred3, cred4}, + deleteValueFunc: func(name string) error { + if name == cred1 || name == cred3 { + return fmt.Errorf("dummy error") + } + return nil + }, + } + defer setRegistryCreateKeyFunc(t, key)() + + errors := removeAllGMSARegistryValues() + + assert.Equal(t, 2, len(errors)) + for _, err := range errors { + assert.Contains(t, err.Error(), "unable to remove registry value") + } + assert.Equal(t, []string{cred1, cred2, cred3, cred4}, key.deleteValueArgs) + assert.Equal(t, []int{0}, key.readValueNamesArgs) + assert.True(t, key.closed) + }) + t.Run("if there's an error opening the registry key", func(t *testing.T) { + defer setRegistryCreateKeyFunc(t, &dummyRegistryKey{}, fmt.Errorf("dummy error"))() + + errors := removeAllGMSARegistryValues() + + require.Equal(t, 1, len(errors)) + assert.Contains(t, errors[0].Error(), "unable to open registry key") + }) + t.Run("if it's unable to list the registry values", func(t *testing.T) { + key := &dummyRegistryKey{readValueNamesError: fmt.Errorf("dummy error")} + defer setRegistryCreateKeyFunc(t, key)() + + errors := removeAllGMSARegistryValues() + + if assert.Equal(t, 1, len(errors)) { + assert.Contains(t, errors[0].Error(), "unable to list values under registry key") + } + assert.True(t, key.closed) + }) +} + +// setRegistryCreateKeyFunc replaces the registryCreateKeyFunc package variable, and returns a function +// to be called to revert the change when done with testing. +func setRegistryCreateKeyFunc(t *testing.T, key *dummyRegistryKey, err ...error) func() { + previousRegistryCreateKeyFunc := registryCreateKeyFunc + + registryCreateKeyFunc = func(baseKey registry.Key, path string, access uint32) (registryKey, bool, error) { + // this should always be called with exactly the same arguments + assert.Equal(t, registry.LOCAL_MACHINE, baseKey) + assert.Equal(t, credentialSpecRegistryLocation, path) + assert.Equal(t, uint32(registry.SET_VALUE), access) + + if len(err) > 0 { + return nil, false, err[0] + } + return key, false, nil + } + + return func() { + registryCreateKeyFunc = previousRegistryCreateKeyFunc + } +} + +// setRandomReader replaces the randomReader package variable with a dummy reader that returns the provided +// byte slice, and returns a function to be called to revert the change when done with testing. +func setRandomReader(b []byte) func() { + previousRandomReader := randomReader + randomReader = bytes.NewReader(b) + return func() { + randomReader = previousRandomReader + } +} diff --git a/pkg/kubelet/dockershim/docker_service.go b/pkg/kubelet/dockershim/docker_service.go index 1baa9a50183..bf487ef2168 100644 --- a/pkg/kubelet/dockershim/docker_service.go +++ b/pkg/kubelet/dockershim/docker_service.go @@ -395,6 +395,8 @@ func (ds *dockerService) GetPodPortMappings(podSandboxID string) ([]*hostport.Po // Start initializes and starts components in dockerService. func (ds *dockerService) Start() error { + ds.initCleanup() + // Initialize the legacy cleanup flag. if ds.startLocalStreamingServer { go func() { @@ -406,6 +408,16 @@ func (ds *dockerService) Start() error { return ds.containerManager.Start() } +// initCleanup is responsible for cleaning up any crufts left by previous +// runs. If there are any errros, it simply logs them. +func (ds *dockerService) initCleanup() { + errors := ds.platformSpecificContainerCreationInitCleanup() + + for _, err := range errors { + klog.Warningf("initialization error: %v", err) + } +} + // Status returns the status of the runtime. func (ds *dockerService) Status(_ context.Context, r *runtimeapi.StatusRequest) (*runtimeapi.StatusResponse, error) { runtimeReady := &runtimeapi.RuntimeCondition{ diff --git a/pkg/kubelet/kuberuntime/BUILD b/pkg/kubelet/kuberuntime/BUILD index 12c16e7e548..c7eec262375 100644 --- a/pkg/kubelet/kuberuntime/BUILD +++ b/pkg/kubelet/kuberuntime/BUILD @@ -89,6 +89,7 @@ go_test( "instrumented_services_test.go", "kuberuntime_container_linux_test.go", "kuberuntime_container_test.go", + "kuberuntime_container_windows_test.go", "kuberuntime_gc_test.go", "kuberuntime_image_test.go", "kuberuntime_manager_test.go", diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container_windows.go b/pkg/kubelet/kuberuntime/kuberuntime_container_windows.go index 77165e81b21..da606c3cfbf 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container_windows.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container_windows.go @@ -23,6 +23,8 @@ import ( "github.com/docker/docker/pkg/sysinfo" "k8s.io/api/core/v1" + utilfeature "k8s.io/apiserver/pkg/util/feature" + kubefeatures "k8s.io/kubernetes/pkg/features" kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" "k8s.io/kubernetes/pkg/securitycontext" @@ -35,6 +37,10 @@ func (m *kubeGenericRuntimeManager) applyPlatformSpecificContainerConfig(config return err } + if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.WindowsGMSA) { + determineEffectiveSecurityContext(config, container, pod) + } + config.Windows = windowsConfig return nil } @@ -97,3 +103,40 @@ func (m *kubeGenericRuntimeManager) generateWindowsContainerConfig(container *v1 return wc, nil } + +const ( + // GMSASpecContainerAnnotationKey is the container annotation where we store the contents of the GMSA credential spec to use. + GMSASpecContainerAnnotationKey = "container.alpha.windows.kubernetes.io/gmsa-credential-spec" + // gMSAContainerSpecPodAnnotationKeySuffix is the suffix of the pod annotation where the GMSA webhook admission controller + // stores the contents of the GMSA credential spec for a given container (the full annotation being the container's name + // with this suffix appended). + gMSAContainerSpecPodAnnotationKeySuffix = "." + GMSASpecContainerAnnotationKey + // gMSAPodSpecPodAnnotationKey is the pod annotation where the GMSA webhook admission controller stores the contents of the GMSA + // credential spec to use for containers that do not have their own specific GMSA cred spec set via a + // gMSAContainerSpecPodAnnotationKeySuffix annotation as explained above + gMSAPodSpecPodAnnotationKey = "pod.alpha.windows.kubernetes.io/gmsa-credential-spec" +) + +// determineEffectiveSecurityContext determines the effective GMSA credential spec and, if any, copies it to the container's +// GMSASpecContainerAnnotationKey annotation. +func determineEffectiveSecurityContext(config *runtimeapi.ContainerConfig, container *v1.Container, pod *v1.Pod) { + var containerCredSpec string + + containerGMSAPodAnnotation := container.Name + gMSAContainerSpecPodAnnotationKeySuffix + if pod.Annotations[containerGMSAPodAnnotation] != "" { + containerCredSpec = pod.Annotations[containerGMSAPodAnnotation] + } else if pod.Annotations[gMSAPodSpecPodAnnotationKey] != "" { + containerCredSpec = pod.Annotations[gMSAPodSpecPodAnnotationKey] + } + + if containerCredSpec != "" { + if config.Annotations == nil { + config.Annotations = make(map[string]string) + } + config.Annotations[GMSASpecContainerAnnotationKey] = containerCredSpec + } else { + // the annotation shouldn't be present, but let's err on the side of caution: + // it should only be set here and nowhere else + delete(config.Annotations, GMSASpecContainerAnnotationKey) + } +} diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container_windows_test.go b/pkg/kubelet/kuberuntime/kuberuntime_container_windows_test.go new file mode 100644 index 00000000000..b438e44acdd --- /dev/null +++ b/pkg/kubelet/kuberuntime/kuberuntime_container_windows_test.go @@ -0,0 +1,82 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kuberuntime + +import ( + "github.com/stretchr/testify/assert" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" +) + +func TestDetermineEffectiveSecurityContext(t *testing.T) { + containerName := "container_name" + container := &corev1.Container{Name: containerName} + dummyCredSpec := "test cred spec contents" + + buildPod := func(annotations map[string]string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + }, + } + } + + t.Run("when there's a specific GMSA for that container, and no pod-wide GMSA", func(t *testing.T) { + containerConfig := &runtimeapi.ContainerConfig{} + + pod := buildPod(map[string]string{ + "container_name.container.alpha.windows.kubernetes.io/gmsa-credential-spec": dummyCredSpec, + }) + + determineEffectiveSecurityContext(containerConfig, container, pod) + + assert.Equal(t, dummyCredSpec, containerConfig.Annotations["container.alpha.windows.kubernetes.io/gmsa-credential-spec"]) + }) + t.Run("when there's a specific GMSA for that container, and a pod-wide GMSA", func(t *testing.T) { + containerConfig := &runtimeapi.ContainerConfig{} + + pod := buildPod(map[string]string{ + "container_name.container.alpha.windows.kubernetes.io/gmsa-credential-spec": dummyCredSpec, + "pod.alpha.windows.kubernetes.io/gmsa-credential-spec": "should be ignored", + }) + + determineEffectiveSecurityContext(containerConfig, container, pod) + + assert.Equal(t, dummyCredSpec, containerConfig.Annotations["container.alpha.windows.kubernetes.io/gmsa-credential-spec"]) + }) + t.Run("when there's no specific GMSA for that container, and a pod-wide GMSA", func(t *testing.T) { + containerConfig := &runtimeapi.ContainerConfig{} + + pod := buildPod(map[string]string{ + "pod.alpha.windows.kubernetes.io/gmsa-credential-spec": dummyCredSpec, + }) + + determineEffectiveSecurityContext(containerConfig, container, pod) + + assert.Equal(t, dummyCredSpec, containerConfig.Annotations["container.alpha.windows.kubernetes.io/gmsa-credential-spec"]) + }) + t.Run("when there's no specific GMSA for that container, and no pod-wide GMSA", func(t *testing.T) { + containerConfig := &runtimeapi.ContainerConfig{} + + determineEffectiveSecurityContext(containerConfig, container, &corev1.Pod{}) + + assert.Nil(t, containerConfig.Annotations) + }) +}