diff --git a/api/api-rules/violation_exceptions.list b/api/api-rules/violation_exceptions.list index 3e07896b676..65aea40c25e 100644 --- a/api/api-rules/violation_exceptions.list +++ b/api/api-rules/violation_exceptions.list @@ -385,6 +385,10 @@ API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1,Policy,Pri API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1,RequestedToCapacityRatioArguments,Resources API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1,RequestedToCapacityRatioArguments,Shape API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1,ServiceAffinity,Labels +API rule violation: list_type_missing,k8s.io/kubelet/config/v1alpha1,CredentialProvider,Args +API rule violation: list_type_missing,k8s.io/kubelet/config/v1alpha1,CredentialProvider,Env +API rule violation: list_type_missing,k8s.io/kubelet/config/v1alpha1,CredentialProvider,MatchImages +API rule violation: list_type_missing,k8s.io/kubelet/config/v1alpha1,CredentialProviderConfig,Providers API rule violation: list_type_missing,k8s.io/kubelet/config/v1beta1,KubeletConfiguration,AllowedUnsafeSysctls API rule violation: list_type_missing,k8s.io/kubelet/config/v1beta1,KubeletConfiguration,ClusterDNS API rule violation: list_type_missing,k8s.io/kubelet/config/v1beta1,KubeletConfiguration,EnforceNodeAllocatable diff --git a/build/kazel_generated.bzl b/build/kazel_generated.bzl index 2414ce7c651..504ec608ea8 100644 --- a/build/kazel_generated.bzl +++ b/build/kazel_generated.bzl @@ -101,6 +101,7 @@ tags_values_pkgs = {"openapi-gen": { "staging/src/k8s.io/kube-proxy/config/v1alpha1", "staging/src/k8s.io/kube-scheduler/config/v1", "staging/src/k8s.io/kube-scheduler/config/v1beta1", + "staging/src/k8s.io/kubelet/config/v1alpha1", "staging/src/k8s.io/kubelet/config/v1beta1", "staging/src/k8s.io/metrics/pkg/apis/custom_metrics/v1beta1", "staging/src/k8s.io/metrics/pkg/apis/custom_metrics/v1beta2", @@ -188,6 +189,7 @@ tags_pkgs_values = {"openapi-gen": { "staging/src/k8s.io/kube-proxy/config/v1alpha1": ["true"], "staging/src/k8s.io/kube-scheduler/config/v1": ["true"], "staging/src/k8s.io/kube-scheduler/config/v1beta1": ["true"], + "staging/src/k8s.io/kubelet/config/v1alpha1": ["true"], "staging/src/k8s.io/kubelet/config/v1beta1": ["true"], "staging/src/k8s.io/metrics/pkg/apis/custom_metrics/v1beta1": ["true"], "staging/src/k8s.io/metrics/pkg/apis/custom_metrics/v1beta2": ["true"], diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 51d555ab216..a3b6fbf04ac 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -1130,6 +1130,8 @@ func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencie kubeServer.CloudProvider, kubeServer.CertDirectory, kubeServer.RootDirectory, + kubeServer.ImageCredentialProviderConfigFile, + kubeServer.ImageCredentialProviderBinDir, kubeServer.RegisterNode, kubeServer.RegisterWithTaints, kubeServer.AllowedUnsafeSysctls, @@ -1204,6 +1206,8 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, cloudProvider string, certDirectory string, rootDirectory string, + imageCredentialProviderConfigFile string, + imageCredentialProviderBinDir string, registerNode bool, registerWithTaints []api.Taint, allowedUnsafeSysctls []string, @@ -1235,6 +1239,8 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, cloudProvider, certDirectory, rootDirectory, + imageCredentialProviderConfigFile, + imageCredentialProviderBinDir, registerNode, registerWithTaints, allowedUnsafeSysctls, diff --git a/hack/.golint_failures b/hack/.golint_failures index afa46d53cfe..f2c49e8fafd 100644 --- a/hack/.golint_failures +++ b/hack/.golint_failures @@ -88,6 +88,7 @@ pkg/controller/volume/persistentvolume pkg/controller/volume/persistentvolume/config/v1alpha1 pkg/controlplane/controller/crdregistration pkg/controlplane/tunneler +pkg/credentialprovider/plugin pkg/features pkg/kubeapiserver pkg/kubectl/cmd/convert @@ -455,6 +456,8 @@ staging/src/k8s.io/kubectl/pkg/polymorphichelpers staging/src/k8s.io/kubectl/pkg/scale staging/src/k8s.io/kubectl/pkg/util/templates staging/src/k8s.io/kubelet/config/v1beta1 +staging/src/k8s.io/kubelet/pkg/apis/credentialprovider +staging/src/k8s.io/kubelet/pkg/apis/credentialprovider/v1alpha1 staging/src/k8s.io/legacy-cloud-providers/vsphere staging/src/k8s.io/metrics/pkg/apis/custom_metrics staging/src/k8s.io/metrics/pkg/apis/custom_metrics/v1beta1 diff --git a/pkg/credentialprovider/BUILD b/pkg/credentialprovider/BUILD index 4485ef3b127..461f5850b07 100644 --- a/pkg/credentialprovider/BUILD +++ b/pkg/credentialprovider/BUILD @@ -46,6 +46,7 @@ filegroup( "//pkg/credentialprovider/aws:all-srcs", "//pkg/credentialprovider/azure:all-srcs", "//pkg/credentialprovider/gcp:all-srcs", + "//pkg/credentialprovider/plugin:all-srcs", "//pkg/credentialprovider/secrets:all-srcs", ], tags = ["automanaged"], diff --git a/pkg/credentialprovider/keyring.go b/pkg/credentialprovider/keyring.go index 494770c0e2f..5ce467f9a8b 100644 --- a/pkg/credentialprovider/keyring.go +++ b/pkg/credentialprovider/keyring.go @@ -158,9 +158,10 @@ func isDefaultRegistryMatch(image string) bool { return !strings.ContainsAny(parts[0], ".:") } +// ParseSchemelessURL parses a schemeless url and returns a url.URL // url.Parse require a scheme, but ours don't have schemes. Adding a // scheme to make url.Parse happy, then clear out the resulting scheme. -func parseSchemelessURL(schemelessURL string) (*url.URL, error) { +func ParseSchemelessURL(schemelessURL string) (*url.URL, error) { parsed, err := url.Parse("https://" + schemelessURL) if err != nil { return nil, err @@ -170,8 +171,8 @@ func parseSchemelessURL(schemelessURL string) (*url.URL, error) { return parsed, nil } -// split the host name into parts, as well as the port -func splitURL(url *url.URL) (parts []string, port string) { +// SplitURL splits the host name into parts, as well as the port +func SplitURL(url *url.URL) (parts []string, port string) { host, port, err := net.SplitHostPort(url.Host) if err != nil { // could not parse port @@ -180,20 +181,20 @@ func splitURL(url *url.URL) (parts []string, port string) { return strings.Split(host, "."), port } -// overloaded version of urlsMatch, operating on strings instead of URLs. -func urlsMatchStr(glob string, target string) (bool, error) { - globURL, err := parseSchemelessURL(glob) +// URLsMatchStr is wrapper for URLsMatch, operating on strings instead of URLs. +func URLsMatchStr(glob string, target string) (bool, error) { + globURL, err := ParseSchemelessURL(glob) if err != nil { return false, err } - targetURL, err := parseSchemelessURL(target) + targetURL, err := ParseSchemelessURL(target) if err != nil { return false, err } - return urlsMatch(globURL, targetURL) + return URLsMatch(globURL, targetURL) } -// check whether the given target url matches the glob url, which may have +// URLsMatch checks whether the given target url matches the glob url, which may have // glob wild cards in the host name. // // Examples: @@ -201,9 +202,9 @@ func urlsMatchStr(glob string, target string) (bool, error) { // globURL=*.docker.io, targetURL=not.right.io => no match // // Note that we don't support wildcards in ports and paths yet. -func urlsMatch(globURL *url.URL, targetURL *url.URL) (bool, error) { - globURLParts, globPort := splitURL(globURL) - targetURLParts, targetPort := splitURL(targetURL) +func URLsMatch(globURL *url.URL, targetURL *url.URL) (bool, error) { + globURLParts, globPort := SplitURL(globURL) + targetURLParts, targetPort := SplitURL(targetURL) if globPort != targetPort { // port doesn't match return false, nil @@ -240,7 +241,7 @@ func (dk *BasicDockerKeyring) Lookup(image string) ([]AuthConfig, bool) { for _, k := range dk.index { // both k and image are schemeless URLs because even though schemes are allowed // in the credential configurations, we remove them in Add. - if matched, _ := urlsMatchStr(k, image); matched { + if matched, _ := URLsMatchStr(k, image); matched { ret = append(ret, dk.creds[k]...) } } diff --git a/pkg/credentialprovider/keyring_test.go b/pkg/credentialprovider/keyring_test.go index 8b5893a201d..3dd7bc2410d 100644 --- a/pkg/credentialprovider/keyring_test.go +++ b/pkg/credentialprovider/keyring_test.go @@ -23,7 +23,7 @@ import ( "testing" ) -func TestUrlsMatch(t *testing.T) { +func TestURLsMatch(t *testing.T) { tests := []struct { globURL string targetURL string @@ -112,7 +112,7 @@ func TestUrlsMatch(t *testing.T) { }, } for _, test := range tests { - matched, _ := urlsMatchStr(test.globURL, test.targetURL) + matched, _ := URLsMatchStr(test.globURL, test.targetURL) if matched != test.matchExpected { t.Errorf("Expected match result of %s and %s to be %t, but was %t", test.globURL, test.targetURL, test.matchExpected, matched) diff --git a/pkg/credentialprovider/plugin/BUILD b/pkg/credentialprovider/plugin/BUILD new file mode 100644 index 00000000000..6362b35b6e8 --- /dev/null +++ b/pkg/credentialprovider/plugin/BUILD @@ -0,0 +1,58 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "config.go", + "plugin.go", + ], + importpath = "k8s.io/kubernetes/pkg/credentialprovider/plugin", + visibility = ["//visibility:public"], + deps = [ + "//pkg/credentialprovider:go_default_library", + "//pkg/kubelet/apis/config:go_default_library", + "//pkg/kubelet/apis/config/v1alpha1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", + "//staging/src/k8s.io/client-go/tools/cache:go_default_library", + "//staging/src/k8s.io/kubelet/pkg/apis/credentialprovider:go_default_library", + "//staging/src/k8s.io/kubelet/pkg/apis/credentialprovider/install:go_default_library", + "//staging/src/k8s.io/kubelet/pkg/apis/credentialprovider/v1alpha1:go_default_library", + "//vendor/k8s.io/klog/v2:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) + +go_test( + name = "go_default_test", + srcs = [ + "config_test.go", + "plugin_test.go", + ], + embed = [":go_default_library"], + deps = [ + "//pkg/credentialprovider:go_default_library", + "//pkg/kubelet/apis/config:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/client-go/tools/cache:go_default_library", + "//staging/src/k8s.io/kubelet/pkg/apis/credentialprovider:go_default_library", + "//staging/src/k8s.io/kubelet/pkg/apis/credentialprovider/v1alpha1:go_default_library", + ], +) diff --git a/pkg/credentialprovider/plugin/config.go b/pkg/credentialprovider/plugin/config.go new file mode 100644 index 00000000000..3857853de5b --- /dev/null +++ b/pkg/credentialprovider/plugin/config.go @@ -0,0 +1,128 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "fmt" + "io/ioutil" + "strings" + + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/kubernetes/pkg/credentialprovider" + kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" +) + +// readCredentialProviderConfigFile receives a path to a config file and decodes it +// into the internal CredentialProviderConfig type. +func readCredentialProviderConfigFile(configPath string) (*kubeletconfig.CredentialProviderConfig, error) { + if configPath == "" { + return nil, fmt.Errorf("credential provider config path is empty") + } + + data, err := ioutil.ReadFile(configPath) + if err != nil { + return nil, fmt.Errorf("unable to read external registry credential provider configuration from %q: %v", configPath, err) + } + + config, err := decode(data) + if err != nil { + return nil, fmt.Errorf("error decoding config %s: %v", configPath, err) + } + + return config, nil +} + +// decode decodes data into the internal CredentialProviderConfig type. +func decode(data []byte) (*kubeletconfig.CredentialProviderConfig, error) { + obj, gvk, err := codecs.UniversalDecoder().Decode(data, nil, nil) + if err != nil { + return nil, err + } + + if gvk.Kind != "CredentialProviderConfig" { + return nil, fmt.Errorf("failed to decode %q (wrong Kind)", gvk.Kind) + } + + if gvk.Group != kubeletconfig.GroupName { + return nil, fmt.Errorf("failed to decode CredentialProviderConfig, unexpected Group: %s", gvk.Group) + } + + if internalConfig, ok := obj.(*kubeletconfig.CredentialProviderConfig); ok { + return internalConfig, nil + } + + return nil, fmt.Errorf("unable to convert %T to *CredentialProviderConfig", obj) +} + +// validateCredentialProviderConfig validates CredentialProviderConfig. +func validateCredentialProviderConfig(config *kubeletconfig.CredentialProviderConfig) field.ErrorList { + allErrs := field.ErrorList{} + + if len(config.Providers) == 0 { + allErrs = append(allErrs, field.Required(field.NewPath("providers"), "at least 1 item in plugins is required")) + } + + fieldPath := field.NewPath("providers") + for _, provider := range config.Providers { + if strings.Contains(provider.Name, "/") { + allErrs = append(allErrs, field.Invalid(fieldPath.Child("name"), provider.Name, "provider name cannot contain '/'")) + } + + if strings.Contains(provider.Name, " ") { + allErrs = append(allErrs, field.Invalid(fieldPath.Child("name"), provider.Name, "provider name cannot contain spaces")) + } + + if provider.Name == "." { + allErrs = append(allErrs, field.Invalid(fieldPath.Child("name"), provider.Name, "provider name cannot be '.'")) + } + + if provider.Name == ".." { + allErrs = append(allErrs, field.Invalid(fieldPath.Child("name"), provider.Name, "provider name cannot be '..'")) + } + + if provider.APIVersion == "" { + allErrs = append(allErrs, field.Required(fieldPath.Child("apiVersion"), "apiVersion is required")) + } else if _, ok := apiVersions[provider.APIVersion]; !ok { + validAPIVersions := []string{} + for apiVersion := range apiVersions { + validAPIVersions = append(validAPIVersions, apiVersion) + } + + allErrs = append(allErrs, field.NotSupported(fieldPath.Child("apiVersion"), provider.APIVersion, validAPIVersions)) + } + + if len(provider.MatchImages) == 0 { + allErrs = append(allErrs, field.Required(fieldPath.Child("matchImages"), "at least 1 item in matchImages is required")) + } + + for _, matchImage := range provider.MatchImages { + if _, err := credentialprovider.ParseSchemelessURL(matchImage); err != nil { + allErrs = append(allErrs, field.Invalid(fieldPath.Child("matchImages"), matchImage, fmt.Sprintf("match image is invalid: %s", err.Error()))) + } + } + + if provider.DefaultCacheDuration == nil { + allErrs = append(allErrs, field.Required(fieldPath.Child("defaultCacheDuration"), "defaultCacheDuration is required")) + } + + if provider.DefaultCacheDuration != nil && provider.DefaultCacheDuration.Duration < 0 { + allErrs = append(allErrs, field.Invalid(fieldPath.Child("defaultCacheDuration"), provider.DefaultCacheDuration.Duration, "defaultCacheDuration must be greater than or equal to 0")) + } + } + + return allErrs +} diff --git a/pkg/credentialprovider/plugin/config_test.go b/pkg/credentialprovider/plugin/config_test.go new file mode 100644 index 00000000000..c8ba3a04e6b --- /dev/null +++ b/pkg/credentialprovider/plugin/config_test.go @@ -0,0 +1,435 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "io/ioutil" + "os" + "reflect" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" +) + +func Test_readCredentialProviderConfigFile(t *testing.T) { + testcases := []struct { + name string + configData string + config *kubeletconfig.CredentialProviderConfig + expectErr bool + }{ + { + name: "config with 1 plugin and 1 image matcher", + configData: `--- +kind: CredentialProviderConfig +apiVersion: kubelet.config.k8s.io/v1alpha1 +providers: + - name: test + matchImages: + - "registry.io/foobar" + defaultCacheDuration: 10m + apiVersion: credentialprovider.kubelet.k8s.io/v1alpha1 + args: + - --v=5 + env: + - name: FOO + value: BAR`, + config: &kubeletconfig.CredentialProviderConfig{ + Providers: []kubeletconfig.CredentialProvider{ + { + Name: "test", + MatchImages: []string{"registry.io/foobar"}, + DefaultCacheDuration: &metav1.Duration{Duration: 10 * time.Minute}, + APIVersion: "credentialprovider.kubelet.k8s.io/v1alpha1", + Args: []string{"--v=5"}, + Env: []kubeletconfig.ExecEnvVar{ + { + Name: "FOO", + Value: "BAR", + }, + }, + }, + }, + }, + }, + { + name: "config with 1 plugin and a wildcard image match", + configData: `--- +kind: CredentialProviderConfig +apiVersion: kubelet.config.k8s.io/v1alpha1 +providers: + - name: test + matchImages: + - "registry.io/*" + defaultCacheDuration: 10m + apiVersion: credentialprovider.kubelet.k8s.io/v1alpha1 + args: + - --v=5 + env: + - name: FOO + value: BAR`, + config: &kubeletconfig.CredentialProviderConfig{ + Providers: []kubeletconfig.CredentialProvider{ + { + Name: "test", + MatchImages: []string{"registry.io/*"}, + DefaultCacheDuration: &metav1.Duration{Duration: 10 * time.Minute}, + APIVersion: "credentialprovider.kubelet.k8s.io/v1alpha1", + Args: []string{"--v=5"}, + Env: []kubeletconfig.ExecEnvVar{ + { + Name: "FOO", + Value: "BAR", + }, + }, + }, + }, + }, + }, + { + name: "config with 1 plugin and multiple image matchers", + configData: `--- +kind: CredentialProviderConfig +apiVersion: kubelet.config.k8s.io/v1alpha1 +providers: + - name: test + matchImages: + - "registry.io/*" + - "foobar.registry.io/*" + defaultCacheDuration: 10m + apiVersion: credentialprovider.kubelet.k8s.io/v1alpha1 + args: + - --v=5 + env: + - name: FOO + value: BAR`, + config: &kubeletconfig.CredentialProviderConfig{ + Providers: []kubeletconfig.CredentialProvider{ + { + Name: "test", + MatchImages: []string{"registry.io/*", "foobar.registry.io/*"}, + DefaultCacheDuration: &metav1.Duration{Duration: 10 * time.Minute}, + APIVersion: "credentialprovider.kubelet.k8s.io/v1alpha1", + Args: []string{"--v=5"}, + Env: []kubeletconfig.ExecEnvVar{ + { + Name: "FOO", + Value: "BAR", + }, + }, + }, + }, + }, + }, + { + name: "config with multiple providers", + configData: `--- +kind: CredentialProviderConfig +apiVersion: kubelet.config.k8s.io/v1alpha1 +providers: + - name: test1 + matchImages: + - "registry.io/one" + defaultCacheDuration: 10m + apiVersion: credentialprovider.kubelet.k8s.io/v1alpha1 + - name: test2 + matchImages: + - "registry.io/two" + defaultCacheDuration: 10m + apiVersion: credentialprovider.kubelet.k8s.io/v1alpha1 + args: + - --v=5 + env: + - name: FOO + value: BAR`, + + config: &kubeletconfig.CredentialProviderConfig{ + Providers: []kubeletconfig.CredentialProvider{ + { + Name: "test1", + MatchImages: []string{"registry.io/one"}, + DefaultCacheDuration: &metav1.Duration{Duration: 10 * time.Minute}, + APIVersion: "credentialprovider.kubelet.k8s.io/v1alpha1", + }, + { + Name: "test2", + MatchImages: []string{"registry.io/two"}, + DefaultCacheDuration: &metav1.Duration{Duration: 10 * time.Minute}, + APIVersion: "credentialprovider.kubelet.k8s.io/v1alpha1", + Args: []string{"--v=5"}, + Env: []kubeletconfig.ExecEnvVar{ + { + Name: "FOO", + Value: "BAR", + }, + }, + }, + }, + }, + }, + { + name: "config with wrong Kind", + configData: `--- +kind: WrongKind +apiVersion: kubelet.config.k8s.io/v1alpha1 +providers: + - name: test + matchImages: + - "registry.io/foobar" + defaultCacheDuration: 10m + apiVersion: credentialprovider.kubelet.k8s.io/v1alpha1 + args: + - --v=5 + env: + - name: FOO + value: BAR`, + config: nil, + expectErr: true, + }, + { + name: "config with wrong apiversion", + configData: `--- +kind: CredentialProviderConfig +apiVersion: foobar/v1alpha1 +providers: + - name: test + matchImages: + - "registry.io/foobar" + defaultCacheDuration: 10m + apiVersion: credentialprovider.kubelet.k8s.io/v1alpha1 + args: + - --v=5 + env: + - name: FOO + value: BAR`, + config: nil, + expectErr: true, + }, + } + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + file, err := ioutil.TempFile("", "config.yaml") + if err != nil { + t.Fatal(err) + } + defer os.Remove(file.Name()) + + _, err = file.WriteString(testcase.configData) + if err != nil { + t.Fatal(err) + } + + authConfig, err := readCredentialProviderConfigFile(file.Name()) + if err != nil && !testcase.expectErr { + t.Fatal(err) + } + + if err == nil && testcase.expectErr { + t.Error("expected error but got none") + } + + if !reflect.DeepEqual(authConfig, testcase.config) { + t.Logf("actual auth config: %#v", authConfig) + t.Logf("expected auth config: %#v", testcase.config) + t.Error("credential provider config did not match") + } + }) + } +} + +func Test_validateCredentialProviderConfig(t *testing.T) { + testcases := []struct { + name string + config *kubeletconfig.CredentialProviderConfig + shouldErr bool + }{ + { + name: "no providers provided", + config: &kubeletconfig.CredentialProviderConfig{}, + shouldErr: true, + }, + { + name: "no matchImages provided", + config: &kubeletconfig.CredentialProviderConfig{ + Providers: []kubeletconfig.CredentialProvider{ + { + Name: "foobar", + MatchImages: []string{}, + DefaultCacheDuration: &metav1.Duration{Duration: time.Minute}, + APIVersion: "credentialprovider.kubelet.k8s.io/v1alpha1", + }, + }, + }, + shouldErr: true, + }, + { + name: "no default cache duration provided", + config: &kubeletconfig.CredentialProviderConfig{ + Providers: []kubeletconfig.CredentialProvider{ + { + Name: "foobar", + MatchImages: []string{"foobar.registry.io"}, + APIVersion: "credentialprovider.kubelet.k8s.io/v1alpha1", + }, + }, + }, + shouldErr: true, + }, + { + name: "name contains '/'", + config: &kubeletconfig.CredentialProviderConfig{ + Providers: []kubeletconfig.CredentialProvider{ + { + Name: "foo/../bar", + MatchImages: []string{"foobar.registry.io"}, + DefaultCacheDuration: &metav1.Duration{Duration: time.Minute}, + APIVersion: "credentialprovider.kubelet.k8s.io/v1alpha1", + }, + }, + }, + shouldErr: true, + }, + { + name: "name is '.'", + config: &kubeletconfig.CredentialProviderConfig{ + Providers: []kubeletconfig.CredentialProvider{ + { + Name: ".", + MatchImages: []string{"foobar.registry.io"}, + DefaultCacheDuration: &metav1.Duration{Duration: time.Minute}, + APIVersion: "credentialprovider.kubelet.k8s.io/v1alpha1", + }, + }, + }, + shouldErr: true, + }, + { + name: "name is '..'", + config: &kubeletconfig.CredentialProviderConfig{ + Providers: []kubeletconfig.CredentialProvider{ + { + Name: "..", + MatchImages: []string{"foobar.registry.io"}, + DefaultCacheDuration: &metav1.Duration{Duration: time.Minute}, + APIVersion: "credentialprovider.kubelet.k8s.io/v1alpha1", + }, + }, + }, + shouldErr: true, + }, + { + name: "name contains spaces", + config: &kubeletconfig.CredentialProviderConfig{ + Providers: []kubeletconfig.CredentialProvider{ + { + Name: "foo bar", + MatchImages: []string{"foobar.registry.io"}, + DefaultCacheDuration: &metav1.Duration{Duration: time.Minute}, + APIVersion: "credentialprovider.kubelet.k8s.io/v1alpha1", + }, + }, + }, + shouldErr: true, + }, + { + name: "no apiVersion", + config: &kubeletconfig.CredentialProviderConfig{ + Providers: []kubeletconfig.CredentialProvider{ + { + Name: "foobar", + MatchImages: []string{"foobar.registry.io"}, + DefaultCacheDuration: &metav1.Duration{Duration: time.Minute}, + APIVersion: "", + }, + }, + }, + shouldErr: true, + }, + { + name: "invalid apiVersion", + config: &kubeletconfig.CredentialProviderConfig{ + Providers: []kubeletconfig.CredentialProvider{ + { + Name: "foobar", + MatchImages: []string{"foobar.registry.io"}, + DefaultCacheDuration: &metav1.Duration{Duration: time.Minute}, + APIVersion: "credentialprovider.kubelet.k8s.io/v1alpha0", + }, + }, + }, + shouldErr: true, + }, + { + name: "negative default cache duration", + config: &kubeletconfig.CredentialProviderConfig{ + Providers: []kubeletconfig.CredentialProvider{ + { + Name: "foobar", + MatchImages: []string{"foobar.registry.io"}, + DefaultCacheDuration: &metav1.Duration{Duration: -1 * time.Minute}, + APIVersion: "credentialprovider.kubelet.k8s.io/v1alpha1", + }, + }, + }, + shouldErr: true, + }, + { + name: "invalid match image", + config: &kubeletconfig.CredentialProviderConfig{ + Providers: []kubeletconfig.CredentialProvider{ + { + Name: "foobar", + MatchImages: []string{"%invalid%"}, + DefaultCacheDuration: &metav1.Duration{Duration: time.Minute}, + APIVersion: "credentialprovider.kubelet.k8s.io/v1alpha1", + }, + }, + }, + shouldErr: true, + }, + { + name: "valid config", + config: &kubeletconfig.CredentialProviderConfig{ + Providers: []kubeletconfig.CredentialProvider{ + { + Name: "foobar", + MatchImages: []string{"foobar.registry.io"}, + DefaultCacheDuration: &metav1.Duration{Duration: time.Minute}, + APIVersion: "credentialprovider.kubelet.k8s.io/v1alpha1", + }, + }, + }, + shouldErr: false, + }, + } + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + errs := validateCredentialProviderConfig(testcase.config) + + if testcase.shouldErr && len(errs) == 0 { + t.Errorf("expected error but got none") + } else if !testcase.shouldErr && len(errs) > 0 { + t.Errorf("expected no error but received errors: %v", errs.ToAggregate()) + + } + }) + } +} diff --git a/pkg/credentialprovider/plugin/plugin.go b/pkg/credentialprovider/plugin/plugin.go new file mode 100644 index 00000000000..6b1c7463df5 --- /dev/null +++ b/pkg/credentialprovider/plugin/plugin.go @@ -0,0 +1,420 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/runtime/serializer/json" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + credentialproviderapi "k8s.io/kubelet/pkg/apis/credentialprovider" + "k8s.io/kubelet/pkg/apis/credentialprovider/install" + credentialproviderv1alpha1 "k8s.io/kubelet/pkg/apis/credentialprovider/v1alpha1" + "k8s.io/kubernetes/pkg/credentialprovider" + kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" + kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/config/v1alpha1" +) + +const ( + globalCacheKey = "global" +) + +var ( + scheme = runtime.NewScheme() + codecs = serializer.NewCodecFactory(scheme) + + apiVersions = map[string]schema.GroupVersion{ + credentialproviderv1alpha1.SchemeGroupVersion.String(): credentialproviderv1alpha1.SchemeGroupVersion, + } +) + +func init() { + install.Install(scheme) + kubeletconfig.AddToScheme(scheme) + kubeletconfigv1alpha1.AddToScheme(scheme) +} + +// RegisterCredentialProviderPlugins is called from kubelet to register external credential provider +// plugins according to the CredentialProviderConfig config file. +func RegisterCredentialProviderPlugins(pluginConfigFile, pluginBinDir string) error { + if _, err := os.Stat(pluginBinDir); err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("plugin binary directory %s did not exist", pluginBinDir) + } + + return fmt.Errorf("error inspecting binary directory %s: %w", pluginBinDir, err) + } + + credentialProviderConfig, err := readCredentialProviderConfigFile(pluginConfigFile) + if err != nil { + return err + } + + errs := validateCredentialProviderConfig(credentialProviderConfig) + if len(errs) > 0 { + return fmt.Errorf("failed to validate credential provider config: %v", errs.ToAggregate()) + } + + for _, provider := range credentialProviderConfig.Providers { + pluginBin := filepath.Join(pluginBinDir, provider.Name) + if _, err := os.Stat(pluginBin); err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("plugin binary executable %s did not exist", pluginBin) + } + + return fmt.Errorf("error inspecting binary executable %s: %w", pluginBin, err) + } + + plugin, err := newPluginProvider(pluginBinDir, provider) + if err != nil { + return fmt.Errorf("error initializing plugin provider %s: %w", provider.Name, err) + } + + credentialprovider.RegisterCredentialProvider(provider.Name, plugin) + } + + return nil +} + +// newPluginProvider returns a new pluginProvider based on the credential provider config. +func newPluginProvider(pluginBinDir string, provider kubeletconfig.CredentialProvider) (*pluginProvider, error) { + mediaType := "application/json" + info, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), mediaType) + if !ok { + return nil, fmt.Errorf("unsupported media type %q", mediaType) + } + + gv, ok := apiVersions[provider.APIVersion] + if !ok { + return nil, fmt.Errorf("invalid apiVersion: %q", provider.APIVersion) + } + + return &pluginProvider{ + matchImages: provider.MatchImages, + cache: cache.NewExpirationStore(cacheKeyFunc, &cacheExpirationPolicy{}), + defaultCacheDuration: provider.DefaultCacheDuration.Duration, + plugin: &execPlugin{ + name: provider.Name, + apiVersion: provider.APIVersion, + encoder: codecs.EncoderForVersion(info.Serializer, gv), + pluginBinDir: pluginBinDir, + args: provider.Args, + envVars: provider.Env, + }, + }, nil +} + +// pluginProvider is the plugin-based implementation of the DockerConfigProvider interface. +type pluginProvider struct { + sync.Mutex + + // matchImages defines the matching image URLs this plugin should operate against. + // The plugin provider will not return any credentials for images that do not match + // against this list of match URLs. + matchImages []string + + // cache stores DockerConfig entries with an expiration time based on the cache duration + // returned from the credential provider plugin. + cache cache.Store + // defaultCacheDuration is the default duration credentials are cached in-memory if the auth plugin + // response did not provide a cache duration for credentials. + defaultCacheDuration time.Duration + + // plugin is the exec implementation of the credential providing plugin. + plugin Plugin +} + +// cacheEntry is the cache object that will be stored in cache.Store. +type cacheEntry struct { + key string + credentials credentialprovider.DockerConfig + expiresAt time.Time +} + +// cacheKeyFunc extracts AuthEntry.MatchKey as the cache key function for the plugin provider. +func cacheKeyFunc(obj interface{}) (string, error) { + key := obj.(*cacheEntry).key + return key, nil +} + +// cacheExpirationPolicy defines implements cache.ExpirationPolicy, determining expiration based on the expiresAt timestamp. +type cacheExpirationPolicy struct{} + +// IsExpired returns true if the current time is after cacheEntry.expiresAt, which is determined by the +// cache duration returned from the credential provider plugin response. +func (c *cacheExpirationPolicy) IsExpired(entry *cache.TimestampedEntry) bool { + return time.Now().After(entry.Obj.(*cacheEntry).expiresAt) +} + +// Provide returns a credentialprovider.DockerConfig based on the credentials returned +// from cache or the exec plugin. +func (p *pluginProvider) Provide(image string) credentialprovider.DockerConfig { + if !p.isImageAllowed(image) { + return credentialprovider.DockerConfig{} + } + + p.Lock() + defer p.Unlock() + + cachedConfig, found, err := p.getCachedCredentials(image) + if err != nil { + klog.Errorf("Failed to get cached docker config: %v", err) + return credentialprovider.DockerConfig{} + } + + if found { + return cachedConfig + } + + response, err := p.plugin.ExecPlugin(context.Background(), image) + if err != nil { + klog.Errorf("Failed getting credential from external registry credential provider: %v", err) + return credentialprovider.DockerConfig{} + } + + var cacheKey string + switch cacheKeyType := response.CacheKeyType; cacheKeyType { + case credentialproviderapi.ImagePluginCacheKeyType: + cacheKey = image + case credentialproviderapi.RegistryPluginCacheKeyType: + registry := parseRegistry(image) + cacheKey = registry + case credentialproviderapi.GlobalPluginCacheKeyType: + cacheKey = globalCacheKey + default: + klog.Errorf("credential provider plugin did not return a valid cacheKeyType: %q", cacheKeyType) + return credentialprovider.DockerConfig{} + } + + dockerConfig := make(credentialprovider.DockerConfig, len(response.Auth)) + for matchImage, authConfig := range response.Auth { + dockerConfig[matchImage] = credentialprovider.DockerConfigEntry{ + Username: authConfig.Username, + Password: authConfig.Password, + } + } + + // cache duration was explicitly 0 so don't cache this response at all. + if response.CacheDuration != nil && response.CacheDuration.Duration == 0 { + return dockerConfig + } + + var expiresAt time.Time + // nil cache duration means use the default cache duration + if response.CacheDuration == nil { + if p.defaultCacheDuration == 0 { + return dockerConfig + } + + expiresAt = time.Now().Add(p.defaultCacheDuration) + } else { + expiresAt = time.Now().Add(response.CacheDuration.Duration) + } + + cachedEntry := &cacheEntry{ + key: cacheKey, + credentials: dockerConfig, + expiresAt: expiresAt, + } + + if err := p.cache.Add(cachedEntry); err != nil { + klog.Errorf("Error adding auth entry to cache: %v", err) + } + + return dockerConfig +} + +// Enabled always returns true since registration of the plugin via kubelet implies it should be enabled. +func (e *pluginProvider) Enabled() bool { + return true +} + +// isImageAllowed returns true if the image matches against the list of allowed matches by the plugin. +func (p *pluginProvider) isImageAllowed(image string) bool { + for _, matchImage := range p.matchImages { + if matched, _ := credentialprovider.URLsMatchStr(matchImage, image); matched { + return true + } + } + + return false +} + +// getCachedCredentials returns a credentialprovider.DockerConfig if cached from the plugin. +func (p *pluginProvider) getCachedCredentials(image string) (credentialprovider.DockerConfig, bool, error) { + obj, found, err := p.cache.GetByKey(image) + if err != nil { + return nil, false, err + } + + if found { + return obj.(*cacheEntry).credentials, true, nil + } + + registry := parseRegistry(image) + obj, found, err = p.cache.GetByKey(registry) + if err != nil { + return nil, false, err + } + + if found { + return obj.(*cacheEntry).credentials, true, nil + } + + obj, found, err = p.cache.GetByKey(globalCacheKey) + if err != nil { + return nil, false, err + } + + if found { + return obj.(*cacheEntry).credentials, true, nil + } + + return nil, false, nil +} + +// Plugin is the interface calling ExecPlugin. This is mainly for testability +// so tests don't have to actually exec any processes. +type Plugin interface { + ExecPlugin(ctx context.Context, image string) (*credentialproviderapi.CredentialProviderResponse, error) +} + +// execPlugin is the implementation of the Plugin interface that execs a credential provider plugin based +// on it's name provided in CredentialProviderConfig. It is assumed that the executable is available in the +// plugin directory provided by the kubelet. +type execPlugin struct { + name string + apiVersion string + encoder runtime.Encoder + args []string + envVars []kubeletconfig.ExecEnvVar + pluginBinDir string +} + +// ExecPlugin executes the plugin binary with arguments and environment variables specified in CredentialProviderConfig: +// +// $ ENV_NAME=ENV_VALUE args[0] args[1] << ./staging/src/k8s.io/kubelet ## explicit # k8s.io/kubelet => ./staging/src/k8s.io/kubelet +k8s.io/kubelet/config/v1alpha1 k8s.io/kubelet/config/v1beta1 +k8s.io/kubelet/pkg/apis/credentialprovider +k8s.io/kubelet/pkg/apis/credentialprovider/install +k8s.io/kubelet/pkg/apis/credentialprovider/v1alpha1 k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1 k8s.io/kubelet/pkg/apis/pluginregistration/v1 k8s.io/kubelet/pkg/apis/podresources/v1