fde87329cb
kubeadm uses certificate rotation to replace the initial high-power cert provided in --kubeconfig with a less powerful certificate on the masters. This requires that we pass the contents of the client config certData and keyData down into the cert store to populate the initial client. Add better comments to describe why the flow is required. Add a test that verifies initial cert contents are written to disk. Change the cert manager to not use MustRegister for prometheus so that it can be tested.
378 lines
10 KiB
Go
378 lines
10 KiB
Go
/*
|
|
Copyright 2016 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package app
|
|
|
|
import (
|
|
"crypto/ecdsa"
|
|
"crypto/elliptic"
|
|
"crypto/rand"
|
|
"crypto/x509"
|
|
"crypto/x509/pkix"
|
|
"encoding/json"
|
|
"encoding/pem"
|
|
"io/ioutil"
|
|
"math/big"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
cfsslconfig "github.com/cloudflare/cfssl/config"
|
|
cfsslsigner "github.com/cloudflare/cfssl/signer"
|
|
cfssllocal "github.com/cloudflare/cfssl/signer/local"
|
|
|
|
certapi "k8s.io/api/certificates/v1beta1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
restclient "k8s.io/client-go/rest"
|
|
certutil "k8s.io/client-go/util/cert"
|
|
)
|
|
|
|
// Test_buildClientCertificateManager validates that we can build a local client cert
|
|
// manager that will use the bootstrap client until we get a valid cert, then use our
|
|
// provided identity on subsequent requests.
|
|
func Test_buildClientCertificateManager(t *testing.T) {
|
|
testDir, err := ioutil.TempDir("", "kubeletcert")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { os.RemoveAll(testDir) }()
|
|
|
|
serverPrivateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
serverCA, err := certutil.NewSelfSignedCACert(certutil.Config{
|
|
CommonName: "the-test-framework",
|
|
}, serverPrivateKey)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
server := &csrSimulator{
|
|
t: t,
|
|
serverPrivateKey: serverPrivateKey,
|
|
serverCA: serverCA,
|
|
}
|
|
s := httptest.NewServer(server)
|
|
defer s.Close()
|
|
|
|
config1 := &restclient.Config{
|
|
UserAgent: "FirstClient",
|
|
Host: s.URL,
|
|
}
|
|
config2 := &restclient.Config{
|
|
UserAgent: "SecondClient",
|
|
Host: s.URL,
|
|
}
|
|
|
|
nodeName := types.NodeName("test")
|
|
m, err := buildClientCertificateManager(config1, config2, testDir, nodeName)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer m.Stop()
|
|
r := m.(rotater)
|
|
|
|
// get an expired CSR (simulating historical output)
|
|
server.backdate = 2 * time.Hour
|
|
server.expectUserAgent = "FirstClient"
|
|
ok, err := r.RotateCerts()
|
|
if !ok || err != nil {
|
|
t.Fatalf("unexpected rotation err: %t %v", ok, err)
|
|
}
|
|
if cert := m.Current(); cert != nil {
|
|
t.Fatalf("Unexpected cert, should be expired: %#v", cert)
|
|
}
|
|
fi := getFileInfo(testDir)
|
|
if len(fi) != 2 {
|
|
t.Fatalf("Unexpected directory contents: %#v", fi)
|
|
}
|
|
|
|
// if m.Current() == nil, then we try again and get a valid
|
|
// client
|
|
server.backdate = 0
|
|
server.expectUserAgent = "FirstClient"
|
|
if ok, err := r.RotateCerts(); !ok || err != nil {
|
|
t.Fatalf("unexpected rotation err: %t %v", ok, err)
|
|
}
|
|
if cert := m.Current(); cert == nil {
|
|
t.Fatalf("Unexpected cert, should be valid: %#v", cert)
|
|
}
|
|
fi = getFileInfo(testDir)
|
|
if len(fi) != 2 {
|
|
t.Fatalf("Unexpected directory contents: %#v", fi)
|
|
}
|
|
|
|
// if m.Current() != nil, then we should use the second client
|
|
server.expectUserAgent = "SecondClient"
|
|
if ok, err := r.RotateCerts(); !ok || err != nil {
|
|
t.Fatalf("unexpected rotation err: %t %v", ok, err)
|
|
}
|
|
if cert := m.Current(); cert == nil {
|
|
t.Fatalf("Unexpected cert, should be valid: %#v", cert)
|
|
}
|
|
fi = getFileInfo(testDir)
|
|
if len(fi) != 2 {
|
|
t.Fatalf("Unexpected directory contents: %#v", fi)
|
|
}
|
|
}
|
|
|
|
func Test_buildClientCertificateManager_populateCertDir(t *testing.T) {
|
|
testDir, err := ioutil.TempDir("", "kubeletcert")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { os.RemoveAll(testDir) }()
|
|
|
|
// when no cert is provided, write nothing to disk
|
|
config1 := &restclient.Config{
|
|
UserAgent: "FirstClient",
|
|
Host: "http://localhost",
|
|
}
|
|
config2 := &restclient.Config{
|
|
UserAgent: "SecondClient",
|
|
Host: "http://localhost",
|
|
}
|
|
nodeName := types.NodeName("test")
|
|
if _, err := buildClientCertificateManager(config1, config2, testDir, nodeName); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fi := getFileInfo(testDir)
|
|
if len(fi) != 0 {
|
|
t.Fatalf("Unexpected directory contents: %#v", fi)
|
|
}
|
|
|
|
// an invalid cert should be ignored
|
|
config2.CertData = []byte("invalid contents")
|
|
config2.KeyData = []byte("invalid contents")
|
|
if _, err := buildClientCertificateManager(config1, config2, testDir, nodeName); err == nil {
|
|
t.Fatal("unexpected non error")
|
|
}
|
|
fi = getFileInfo(testDir)
|
|
if len(fi) != 0 {
|
|
t.Fatalf("Unexpected directory contents: %#v", fi)
|
|
}
|
|
|
|
// an expired client certificate should be written to disk, because the cert manager can
|
|
// use config1 to refresh it and the cert manager won't return it for clients.
|
|
config2.CertData, config2.KeyData = genClientCert(t, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour))
|
|
if _, err := buildClientCertificateManager(config1, config2, testDir, nodeName); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fi = getFileInfo(testDir)
|
|
if len(fi) != 2 {
|
|
t.Fatalf("Unexpected directory contents: %#v", fi)
|
|
}
|
|
|
|
// a valid, non-expired client certificate should be written to disk
|
|
config2.CertData, config2.KeyData = genClientCert(t, time.Now().Add(-time.Hour), time.Now().Add(24*time.Hour))
|
|
if _, err := buildClientCertificateManager(config1, config2, testDir, nodeName); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fi = getFileInfo(testDir)
|
|
if len(fi) != 2 {
|
|
t.Fatalf("Unexpected directory contents: %#v", fi)
|
|
}
|
|
|
|
}
|
|
|
|
func getFileInfo(dir string) map[string]os.FileInfo {
|
|
fi := make(map[string]os.FileInfo)
|
|
filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
|
|
if path == dir {
|
|
return nil
|
|
}
|
|
fi[path] = info
|
|
if !info.IsDir() {
|
|
os.Remove(path)
|
|
}
|
|
return nil
|
|
})
|
|
return fi
|
|
}
|
|
|
|
type rotater interface {
|
|
RotateCerts() (bool, error)
|
|
}
|
|
|
|
func getCSR(req *http.Request) (*certapi.CertificateSigningRequest, error) {
|
|
if req.Body == nil {
|
|
return nil, nil
|
|
}
|
|
body, err := ioutil.ReadAll(req.Body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
csr := &certapi.CertificateSigningRequest{}
|
|
if err := json.Unmarshal(body, csr); err != nil {
|
|
return nil, err
|
|
}
|
|
return csr, nil
|
|
}
|
|
|
|
func mustMarshal(obj interface{}) []byte {
|
|
data, err := json.Marshal(obj)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
return data
|
|
}
|
|
|
|
type csrSimulator struct {
|
|
t *testing.T
|
|
|
|
serverPrivateKey *ecdsa.PrivateKey
|
|
serverCA *x509.Certificate
|
|
backdate time.Duration
|
|
|
|
expectUserAgent string
|
|
|
|
lock sync.Mutex
|
|
csr *certapi.CertificateSigningRequest
|
|
}
|
|
|
|
func (s *csrSimulator) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
|
s.lock.Lock()
|
|
defer s.lock.Unlock()
|
|
t := s.t
|
|
|
|
t.Logf("Request %s %s %s", req.Method, req.URL, req.UserAgent())
|
|
|
|
if len(s.expectUserAgent) > 0 && req.UserAgent() != s.expectUserAgent {
|
|
t.Errorf("Unexpected user agent: %s", req.UserAgent())
|
|
}
|
|
|
|
switch {
|
|
case req.Method == "POST" && req.URL.Path == "/apis/certificates.k8s.io/v1beta1/certificatesigningrequests":
|
|
csr, err := getCSR(req)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if csr.Name == "" {
|
|
csr.Name = "test-csr"
|
|
}
|
|
|
|
csr.UID = types.UID("1")
|
|
csr.ResourceVersion = "1"
|
|
data := mustMarshal(csr)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.Write(data)
|
|
|
|
csr = csr.DeepCopy()
|
|
csr.ResourceVersion = "2"
|
|
var usages []string
|
|
for _, usage := range csr.Spec.Usages {
|
|
usages = append(usages, string(usage))
|
|
}
|
|
policy := &cfsslconfig.Signing{
|
|
Default: &cfsslconfig.SigningProfile{
|
|
Usage: usages,
|
|
Expiry: time.Hour,
|
|
ExpiryString: time.Hour.String(),
|
|
Backdate: s.backdate,
|
|
},
|
|
}
|
|
cfs, err := cfssllocal.NewSigner(s.serverPrivateKey, s.serverCA, cfsslsigner.DefaultSigAlgo(s.serverPrivateKey), policy)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
csr.Status.Certificate, err = cfs.Sign(cfsslsigner.SignRequest{
|
|
Request: string(csr.Spec.Request),
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
csr.Status.Conditions = []certapi.CertificateSigningRequestCondition{
|
|
{Type: certapi.CertificateApproved},
|
|
}
|
|
s.csr = csr
|
|
|
|
case req.Method == "GET" && req.URL.Path == "/apis/certificates.k8s.io/v1beta1/certificatesigningrequests" && req.URL.RawQuery == "fieldSelector=metadata.name%3Dtest-csr&limit=500":
|
|
if s.csr == nil {
|
|
t.Fatalf("no csr")
|
|
}
|
|
csr := s.csr.DeepCopy()
|
|
|
|
data := mustMarshal(&certapi.CertificateSigningRequestList{
|
|
ListMeta: metav1.ListMeta{
|
|
ResourceVersion: "2",
|
|
},
|
|
Items: []certapi.CertificateSigningRequest{
|
|
*csr,
|
|
},
|
|
})
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.Write(data)
|
|
|
|
case req.Method == "GET" && req.URL.Path == "/apis/certificates.k8s.io/v1beta1/certificatesigningrequests" && req.URL.RawQuery == "fieldSelector=metadata.name%3Dtest-csr&resourceVersion=2&watch=true":
|
|
if s.csr == nil {
|
|
t.Fatalf("no csr")
|
|
}
|
|
csr := s.csr.DeepCopy()
|
|
|
|
data := mustMarshal(&metav1.WatchEvent{
|
|
Type: "ADDED",
|
|
Object: runtime.RawExtension{
|
|
Raw: mustMarshal(csr),
|
|
},
|
|
})
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.Write(data)
|
|
|
|
default:
|
|
t.Fatalf("unexpected request: %s %s", req.Method, req.URL)
|
|
}
|
|
}
|
|
|
|
// genClientCert generates an x509 certificate for testing. Certificate and key
|
|
// are returned in PEM encoding.
|
|
func genClientCert(t *testing.T, from, to time.Time) ([]byte, []byte) {
|
|
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
keyRaw, err := x509.MarshalECPrivateKey(key)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
|
|
serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cert := &x509.Certificate{
|
|
SerialNumber: serialNumber,
|
|
Subject: pkix.Name{Organization: []string{"Acme Co"}},
|
|
NotBefore: from,
|
|
NotAfter: to,
|
|
|
|
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
|
|
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
|
|
BasicConstraintsValid: true,
|
|
}
|
|
certRaw, err := x509.CreateCertificate(rand.Reader, cert, cert, key.Public(), key)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certRaw}),
|
|
pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyRaw})
|
|
}
|