github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

When using external CA, look for common trust anchor within CA bundle.

Browse Source
This commit is contained in:
Alex Stundzia
2024-06-28 18:04:31 -05:00
parent 19d9e4fa7f
commit 58ff17bbfa
3 changed files with 232 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go
View File

@@ -264,10 +264,36 @@ func validateKubeConfig(outDir, filename string, config *clientcmdapi.Config) er
}
caExpected := bytes.TrimSpace(config.Clusters[expectedCluster].CertificateAuthorityData)
// If the current CA cert on disk doesn't match the expected CA cert, error out because we have a file, but it's stale
if !bytes.Equal(caCurrent, caExpected) {
return errors.Errorf("a kubeconfig file %q exists already but has got the wrong CA cert", kubeConfigFilePath)
// Parse the current certificate authority data
currentCACerts, err := certutil.ParseCertsPEM(caCurrent)
if err != nil {
return errors.Errorf("the kubeconfig file %q contains an invalid CA cert", kubeConfigFilePath)
}
// Parse the expected certificate authority data
expectedCACerts, err := certutil.ParseCertsPEM(caExpected)
if err != nil {
return errors.Errorf("the expected base64 encoded CA cert could not be parsed as a PEM:\n%s\n", caExpected)
}
// Only use the first certificate in the current CA cert list
currentCaCert := currentCACerts[0]
// Find a common trust anchor
trustAnchorFound := false
for _, expectedCaCert := range expectedCACerts {
// Compare the current CA cert to the expected CA cert.
// If the certificates match then a common trust anchor was found.
if currentCaCert.Equal(expectedCaCert) {
trustAnchorFound = true
break
}
}
if !trustAnchorFound {
return errors.Errorf("a kubeconfig file %q exists but does not contain a trusted CA in its current context's "+
"cluster. Total CA certificates found: %d", kubeConfigFilePath, len(currentCACerts))
}
// If the current API Server location on disk doesn't match the expected API server, show a warning
if currentConfig.Clusters[currentCluster].Server != config.Clusters[expectedCluster].Server {
klog.Warningf("a kubeconfig file %q exists already but has an unexpected API Server URL: expected: %s, got: %s",
@@ -386,12 +412,18 @@ func writeKubeConfigFromSpec(out io.Writer, spec *kubeConfigSpec, clustername st
func ValidateKubeconfigsForExternalCA(outDir string, cfg *kubeadmapi.InitConfiguration) error {
// Creates a kubeconfig file with the target CA and server URL
// to be used as a input for validating user provided kubeconfig files
caCert, err := pkiutil.TryLoadCertFromDisk(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName)
caCert, intermediaries, err := pkiutil.TryLoadCertChainFromDisk(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName)
if err != nil {
return errors.Wrapf(err, "the CA file couldn't be loaded")
}
// Combine caCert and intermediaries into one array
caCertChain := append([]*x509.Certificate{caCert}, intermediaries...)
// Validate period
certsphase.CheckCertificatePeriodValidity(kubeadmconstants.CACertAndKeyBaseName, caCert)
for _, cert := range caCertChain {
certsphase.CheckCertificatePeriodValidity(kubeadmconstants.CACertAndKeyBaseName, cert)
}
// validate user provided kubeconfig files for the scheduler and controller-manager
localAPIEndpoint, err := kubeadmutil.GetLocalAPIEndpoint(&cfg.LocalAPIEndpoint)
@@ -399,7 +431,12 @@ func ValidateKubeconfigsForExternalCA(outDir string, cfg *kubeadmapi.InitConfigu
return err
}
validationConfigLocal := kubeconfigutil.CreateBasic(localAPIEndpoint, "dummy", "dummy", pkiutil.EncodeCertPEM(caCert))
caCertBytes, err := pkiutil.EncodeCertBundlePEM(caCertChain)
if err != nil {
return err
}
validationConfigLocal := kubeconfigutil.CreateBasic(localAPIEndpoint, "dummy", "dummy", caCertBytes)
kubeConfigFileNamesLocal := []string{
kubeadmconstants.ControllerManagerKubeConfigFileName,
kubeadmconstants.SchedulerKubeConfigFileName,
@@ -417,7 +454,7 @@ func ValidateKubeconfigsForExternalCA(outDir string, cfg *kubeadmapi.InitConfigu
return err
}
validationConfigCPE := kubeconfigutil.CreateBasic(controlPlaneEndpoint, "dummy", "dummy", pkiutil.EncodeCertPEM(caCert))
validationConfigCPE := kubeconfigutil.CreateBasic(controlPlaneEndpoint, "dummy", "dummy", caCertBytes)
kubeConfigFileNamesCPE := []string{
kubeadmconstants.AdminKubeConfigFileName,
kubeadmconstants.SuperAdminKubeConfigFileName,

182
cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go
View File

@@ -608,7 +608,11 @@ func TestValidateKubeConfig(t *testing.T) {
func TestValidateKubeconfigsForExternalCA(t *testing.T) {
tmpDir := testutil.SetupTempDir(t)
defer os.RemoveAll(tmpDir)
defer func() {
if err := os.RemoveAll(tmpDir); err != nil {
t.Error(err)
}
}()
pkiDir := filepath.Join(tmpDir, "pki")
initConfig := &kubeadmapi.InitConfiguration{
@@ -623,11 +627,9 @@ func TestValidateKubeconfigsForExternalCA(t *testing.T) {
// creates CA, write to pkiDir and remove ca.key to get into external CA condition
caCert, caKey := certstestutil.SetupCertificateAuthority(t)
if err := pkiutil.WriteCertAndKey(pkiDir, kubeadmconstants.CACertAndKeyBaseName, caCert, caKey); err != nil {
t.Fatalf("failure while saving CA certificate and key: %v", err)
}
if err := os.Remove(filepath.Join(pkiDir, kubeadmconstants.CAKeyName)); err != nil {
t.Fatalf("failure while deleting ca.key: %v", err)
if err := pkiutil.WriteCertBundle(pkiDir, kubeadmconstants.CACertAndKeyBaseName, []*x509.Certificate{caCert}); err != nil {
t.Fatalf("failure while saving CA certificate: %v", err)
}
notAfter, _ := time.Parse(time.RFC3339, "2026-01-02T15:04:05Z")
@@ -697,7 +699,11 @@ func TestValidateKubeconfigsForExternalCA(t *testing.T) {
for name, test := range tests {
t.Run(name, func(t *testing.T) {
tmpdir := testutil.SetupTempDir(t)
defer os.RemoveAll(tmpdir)
defer func() {
if err := os.RemoveAll(tmpdir); err != nil {
t.Error(err)
}
}()
for name, config := range test.filesToWrite {
if err := createKubeConfigFileIfNotExists(tmpdir, name, config); err != nil {
@@ -719,6 +725,166 @@ func TestValidateKubeconfigsForExternalCA(t *testing.T) {
}
}
func TestValidateKubeconfigsForExternalCAMissingRoot(t *testing.T) {
tmpDir := testutil.SetupTempDir(t)
defer func() {
if err := os.RemoveAll(tmpDir); err != nil {
t.Error(err)
}
}()
pkiDir := filepath.Join(tmpDir, "pki")
initConfig := &kubeadmapi.InitConfiguration{
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
CertificatesDir: pkiDir,
},
LocalAPIEndpoint: kubeadmapi.APIEndpoint{
BindPort: 1234,
AdvertiseAddress: "1.2.3.4",
},
}
// Creates CA, write to pkiDir and remove ca.key to get into external CA mode
caCert, caKey := certstestutil.SetupCertificateAuthority(t)
// Setup multiple intermediate certificate authorities (CAs) for testing purposes.
// This is "Root CA" signs "Intermediate Authority 1A" signs "Intermediate Authority 2A"
intermediateCACert1a, intermediateCAKey1a := certstestutil.SetupIntermediateCertificateAuthority(t, caCert, caKey, "Intermediate Authority 1A")
intermediateCACert2a, intermediateCAKey2a := certstestutil.SetupIntermediateCertificateAuthority(t, intermediateCACert1a, intermediateCAKey1a, "Intermediate Authority 1A")
// These two CA certificates should both validate using the Intermediate CA 2B certificate
// This is "Root CA" signs "Intermediate Authority 1B" signs "Intermediate Authority 2B"
intermediateCACert1b, intermediateCAKey1b := certstestutil.SetupIntermediateCertificateAuthority(t, caCert, caKey, "Intermediate Authority 1B")
intermediateCACert2b, intermediateCAKey2b := certstestutil.SetupIntermediateCertificateAuthority(t, intermediateCACert1b, intermediateCAKey1b, "Intermediate Authority 2B")
notAfter, _ := time.Parse(time.RFC3339, "2036-01-02T15:04:05Z")
clusterName := "myOrg1"
var validCaCertBundle []*x509.Certificate
validCaCertBundle = append(validCaCertBundle, caCert, intermediateCACert1a, intermediateCACert2a)
multipleCAConfigRootCAIssuer := setupKubeConfigWithClientAuth(t, caCert, caKey, notAfter, "https://1.2.3.4:1234", "test-cluster", clusterName)
multipleCAConfigIntermediateCA1aIssuer := setupKubeConfigWithClientAuth(t, intermediateCACert1a, intermediateCAKey1a, notAfter, "https://1.2.3.4:1234", "test-cluster", clusterName)
multipleCAConfigIntermediateCA2aIssuer := setupKubeConfigWithClientAuth(t, intermediateCACert2a, intermediateCAKey2a, notAfter, "https://1.2.3.4:1234", "test-cluster", clusterName)
var caBundleMissingRootCA []*x509.Certificate
caBundleMissingRootCA = append(caBundleMissingRootCA, intermediateCACert1b, intermediateCACert2b)
multipleCAConfigNoRootCA := setupKubeConfigWithClientAuth(t, intermediateCACert2b, intermediateCAKey2b, notAfter, "https://1.2.3.4:1234", "test-cluster", clusterName)
multipleCAConfigDifferentIssuer := setupKubeConfigWithClientAuth(t, intermediateCACert2a, intermediateCAKey2a, notAfter, "https://1.2.3.4:1234", "test-cluster", clusterName)
var caBundlePartialChain []*x509.Certificate
caBundlePartialChain = append(caBundlePartialChain, intermediateCACert1a)
multipleCaPartialCA := setupKubeConfigWithClientAuth(t, intermediateCACert2b, intermediateCAKey2b, notAfter, "https://1.2.3.4:1234", "test-cluster", clusterName)
tests := map[string]struct {
filesToWrite map[string]*clientcmdapi.Config
initConfig *kubeadmapi.InitConfiguration
expectedError bool
caCertificate []*x509.Certificate
}{
// Positive test cases
"valid config issued from RootCA": {
filesToWrite: map[string]*clientcmdapi.Config{
kubeadmconstants.AdminKubeConfigFileName: multipleCAConfigRootCAIssuer,
kubeadmconstants.SuperAdminKubeConfigFileName: multipleCAConfigRootCAIssuer,
kubeadmconstants.KubeletKubeConfigFileName: multipleCAConfigRootCAIssuer,
kubeadmconstants.ControllerManagerKubeConfigFileName: multipleCAConfigRootCAIssuer,
kubeadmconstants.SchedulerKubeConfigFileName: multipleCAConfigRootCAIssuer,
},
caCertificate: validCaCertBundle,
initConfig: initConfig,
expectedError: false,
},
"valid config issued from IntermediateCA 1A": {
filesToWrite: map[string]*clientcmdapi.Config{
kubeadmconstants.AdminKubeConfigFileName: multipleCAConfigIntermediateCA1aIssuer,
kubeadmconstants.SuperAdminKubeConfigFileName: multipleCAConfigIntermediateCA1aIssuer,
kubeadmconstants.KubeletKubeConfigFileName: multipleCAConfigIntermediateCA1aIssuer,
kubeadmconstants.ControllerManagerKubeConfigFileName: multipleCAConfigIntermediateCA1aIssuer,
kubeadmconstants.SchedulerKubeConfigFileName: multipleCAConfigIntermediateCA1aIssuer,
},
caCertificate: validCaCertBundle,
initConfig: initConfig,
expectedError: false,
},
"valid config issued from IntermediateCA 2A": {
filesToWrite: map[string]*clientcmdapi.Config{
kubeadmconstants.AdminKubeConfigFileName: multipleCAConfigIntermediateCA2aIssuer,
kubeadmconstants.SuperAdminKubeConfigFileName: multipleCAConfigIntermediateCA2aIssuer,
kubeadmconstants.KubeletKubeConfigFileName: multipleCAConfigIntermediateCA2aIssuer,
kubeadmconstants.ControllerManagerKubeConfigFileName: multipleCAConfigIntermediateCA2aIssuer,
kubeadmconstants.SchedulerKubeConfigFileName: multipleCAConfigIntermediateCA2aIssuer,
},
caCertificate: validCaCertBundle,
initConfig: initConfig,
expectedError: false,
},
"valid config issued from IntermediateCA 2B, CA missing root certificate": {
filesToWrite: map[string]*clientcmdapi.Config{
kubeadmconstants.AdminKubeConfigFileName: multipleCAConfigNoRootCA,
kubeadmconstants.SuperAdminKubeConfigFileName: multipleCAConfigNoRootCA,
kubeadmconstants.KubeletKubeConfigFileName: multipleCAConfigNoRootCA,
kubeadmconstants.ControllerManagerKubeConfigFileName: multipleCAConfigNoRootCA,
kubeadmconstants.SchedulerKubeConfigFileName: multipleCAConfigNoRootCA,
},
caCertificate: caBundleMissingRootCA,
initConfig: initConfig,
expectedError: false,
},
// Negative test cases
"invalid config issued from IntermediateCA 2A, testing a chain with a different issuer": {
filesToWrite: map[string]*clientcmdapi.Config{
kubeadmconstants.AdminKubeConfigFileName: multipleCAConfigDifferentIssuer,
kubeadmconstants.SuperAdminKubeConfigFileName: multipleCAConfigDifferentIssuer,
kubeadmconstants.KubeletKubeConfigFileName: multipleCAConfigDifferentIssuer,
kubeadmconstants.ControllerManagerKubeConfigFileName: multipleCAConfigDifferentIssuer,
kubeadmconstants.SchedulerKubeConfigFileName: multipleCAConfigDifferentIssuer,
},
caCertificate: caBundleMissingRootCA,
initConfig: initConfig,
expectedError: true,
},
"invalid config issued from IntermediateCA 2B chain, CA only contains Intermediate 1A": {
filesToWrite: map[string]*clientcmdapi.Config{
kubeadmconstants.AdminKubeConfigFileName: multipleCaPartialCA,
kubeadmconstants.SuperAdminKubeConfigFileName: multipleCaPartialCA,
kubeadmconstants.KubeletKubeConfigFileName: multipleCaPartialCA,
kubeadmconstants.ControllerManagerKubeConfigFileName: multipleCaPartialCA,
kubeadmconstants.SchedulerKubeConfigFileName: multipleCaPartialCA,
},
caCertificate: caBundlePartialChain,
initConfig: initConfig,
expectedError: true,
},
}
for name, test := range tests {
t.Run(name, func(t *testing.T) {
tmpdir := testutil.SetupTempDir(t)
defer func() {
if err := os.RemoveAll(tmpdir); err != nil {
t.Error(err)
}
}()
for name, config := range test.filesToWrite {
if err := createKubeConfigFileIfNotExists(tmpdir, name, config); err != nil {
t.Errorf("createKubeConfigFileIfNotExists failed: %v", err)
}
}
if err := pkiutil.WriteCertBundle(pkiDir, kubeadmconstants.CACertAndKeyBaseName, test.caCertificate); err != nil {
t.Fatalf("Failure while saving CA certificate: %v", err)
}
err := ValidateKubeconfigsForExternalCA(tmpdir, test.initConfig)
if (err != nil) != test.expectedError {
t.Fatalf("ValidateKubeconfigsForExternalCA failed\n%s\nexpected error: %t\n\tgot: %t\nerror: %v",
name, test.expectedError, (err != nil), err)
}
})
}
}
// setupKubeConfigWithClientAuth is a test utility function that wraps buildKubeConfigFromSpec for building a KubeConfig object With ClientAuth
func setupKubeConfigWithClientAuth(t *testing.T, caCert *x509.Certificate, caKey crypto.Signer, notAfter time.Time, apiServer, clientName, clustername string, organizations ...string) *clientcmdapi.Config {
spec := &kubeConfigSpec{
@@ -740,7 +906,7 @@ func setupKubeConfigWithClientAuth(t *testing.T, caCert *x509.Certificate, caKey
return config
}
// setupKubeConfigWithClientAuth is a test utility function that wraps buildKubeConfigFromSpec for building a KubeConfig object With Token
// setupKubeConfigWithTokenAuth is a test utility function that wraps buildKubeConfigFromSpec for building a KubeConfig object With Token
func setupKubeConfigWithTokenAuth(t *testing.T, caCert *x509.Certificate, apiServer, clientName, token, clustername string) *clientcmdapi.Config {
spec := &kubeConfigSpec{
CACert: caCert,

15
cmd/kubeadm/app/util/certs/util.go
View File

@@ -38,7 +38,20 @@ func SetupCertificateAuthority(t *testing.T) (*x509.Certificate, crypto.Signer)
Config: certutil.Config{CommonName: "kubernetes"},
})
if err != nil {
t.Fatalf("failure while generating CA certificate and key: %v", err)
t.Fatalf("Failure while generating CA certificate and key: %v", err)
}
return caCert, caKey
}
// SetupIntermediateCertificateAuthority is a utility function for kubeadm testing that creates a
// Intermediate CertificateAuthority cert/key pair
func SetupIntermediateCertificateAuthority(t *testing.T, parentCert *x509.Certificate, parentKey crypto.Signer, cn string) (*x509.Certificate, crypto.Signer) {
caCert, caKey, err := pkiutil.NewIntermediateCertificateAuthority(parentCert, parentKey, &pkiutil.CertConfig{
Config: certutil.Config{CommonName: cn},
})
if err != nil {
t.Fatalf("Failure while generating intermediate CA certificate and key: %v", err)
}
return caCert, caKey