Dynamic serving certificate

Reload certificate cert and key file from disk every minute and notify
the dynamic certificate controller when they change, allowing serving
tls config to be updated.

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/BUILD

@@ -5,7 +5,8 @@ go_library(
     srcs = [
         "cert_key.go",
         "client_ca.go",
-        "dynamicfile_content.go",
+        "dynamic_cafile_content.go",
+        "dynamic_serving_content.go",
         "named_certificates.go",
         "static_content.go",
         "tlsconfig.go",

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_serving_content.go

@@ -0,0 +1,179 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"crypto/tls"
+	"fmt"
+	"io/ioutil"
+	"sync/atomic"
+	"time"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog"
+)
+
+// DynamicFileServingContent provides a CertKeyContentProvider that can dynamically react to new file content
+type DynamicFileServingContent struct {
+	name string
+
+	// certFile is the name of the certificate file to read.
+	certFile string
+	// keyFile is the name of the key file to read.
+	keyFile string
+
+	// servingCert is a certKeyContent that contains the last read, non-zero length content of the key and cert
+	servingCert atomic.Value
+
+	listeners []Listener
+
+	// queue only ever has one item, but it has nice error handling backoff/retry semantics
+	queue workqueue.RateLimitingInterface
+}
+
+var _ Notifier = &DynamicFileServingContent{}
+var _ CertKeyContentProvider = &DynamicFileServingContent{}
+var _ ControllerRunner = &DynamicFileServingContent{}
+
+// NewDynamicServingContentFromFiles returns a dynamic CertKeyContentProvider based on a cert and key filename
+func NewDynamicServingContentFromFiles(purpose, certFile, keyFile string) (*DynamicFileServingContent, error) {
+	if len(certFile) == 0 || len(keyFile) == 0 {
+		return nil, fmt.Errorf("missing filename for serving cert")
+	}
+	name := fmt.Sprintf("%s::%s::%s", purpose, certFile, keyFile)
+
+	ret := &DynamicFileServingContent{
+		name:     name,
+		certFile: certFile,
+		keyFile:  keyFile,
+		queue:    workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), fmt.Sprintf("DynamicCABundle-%s", purpose)),
+	}
+	if err := ret.loadServingCert(); err != nil {
+		return nil, err
+	}
+
+	return ret, nil
+}
+
+// AddListener adds a listener to be notified when the serving cert content changes.
+func (c *DynamicFileServingContent) AddListener(listener Listener) {
+	c.listeners = append(c.listeners, listener)
+}
+
+// loadServingCert determines the next set of content for the file.
+func (c *DynamicFileServingContent) loadServingCert() error {
+	cert, err := ioutil.ReadFile(c.certFile)
+	if err != nil {
+		return err
+	}
+	key, err := ioutil.ReadFile(c.keyFile)
+	if err != nil {
+		return err
+	}
+	if len(cert) == 0 || len(key) == 0 {
+		return fmt.Errorf("missing content for serving cert %q", c.Name())
+	}
+
+	// Ensure that the key matches the cert and both are valid
+	_, err = tls.X509KeyPair(cert, key)
+	if err != nil {
+		return err
+	}
+
+	newCertKey := &certKeyContent{
+		cert: cert,
+		key:  key,
+	}
+
+	// check to see if we have a change. If the values are the same, do nothing.
+	existing, ok := c.servingCert.Load().(*certKeyContent)
+	if ok && existing != nil && existing.Equal(newCertKey) {
+		return nil
+	}
+
+	c.servingCert.Store(newCertKey)
+
+	for _, listener := range c.listeners {
+		listener.Enqueue()
+	}
+
+	return nil
+}
+
+// RunOnce runs a single sync loop
+func (c *DynamicFileServingContent) RunOnce() error {
+	return c.loadServingCert()
+}
+
+// Run starts the controller and blocks until stopCh is closed.
+func (c *DynamicFileServingContent) Run(workers int, stopCh <-chan struct{}) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+
+	klog.Infof("Starting %s", c.name)
+	defer klog.Infof("Shutting down %s", c.name)
+
+	// doesn't matter what workers say, only start one.
+	go wait.Until(c.runWorker, time.Second, stopCh)
+
+	// start timer that rechecks every minute, just in case.  this also serves to prime the controller quickly.
+	_ = wait.PollImmediateUntil(FileRefreshDuration, func() (bool, error) {
+		c.queue.Add(workItemKey)
+		return false, nil
+	}, stopCh)
+
+	// TODO this can be wired to an fsnotifier as well.
+
+	<-stopCh
+}
+
+func (c *DynamicFileServingContent) runWorker() {
+	for c.processNextWorkItem() {
+	}
+}
+
+func (c *DynamicFileServingContent) processNextWorkItem() bool {
+	dsKey, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	defer c.queue.Done(dsKey)
+
+	err := c.loadServingCert()
+	if err == nil {
+		c.queue.Forget(dsKey)
+		return true
+	}
+
+	utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err))
+	c.queue.AddRateLimited(dsKey)
+
+	return true
+}
+
+// Name is just an identifier
+func (c *DynamicFileServingContent) Name() string {
+	return c.name
+}
+
+// CurrentCertKeyContent provides serving cert byte content
+func (c *DynamicFileServingContent) CurrentCertKeyContent() ([]byte, []byte) {
+	certKeyContent := c.servingCert.Load().(*certKeyContent)
+	return certKeyContent.cert, certKeyContent.key
+}

staging/src/k8s.io/apiserver/pkg/server/options/serving.go

@@ -226,7 +226,7 @@ func (s *SecureServingOptions) ApplyTo(config **server.SecureServingInfo) error
 	// load main cert
 	if len(serverCertFile) != 0 || len(serverKeyFile) != 0 {
 		var err error
-		c.Cert, err = dynamiccertificates.NewStaticCertKeyContentFromFiles(serverCertFile, serverKeyFile)
+		c.Cert, err = dynamiccertificates.NewDynamicServingContentFromFiles("serving-cert", serverCertFile, serverKeyFile)
 		if err != nil {
 			return err
 		}

staging/src/k8s.io/apiserver/pkg/server/secure_serving.go

@@ -76,6 +76,9 @@ func (s *SecureServingInfo) tlsConfig(stopCh <-chan struct{}) (*tls.Config, erro
 		if notifier, ok := s.ClientCA.(dynamiccertificates.Notifier); ok {
 			notifier.AddListener(dynamicCertificateController)
 		}
+		if notifier, ok := s.Cert.(dynamiccertificates.Notifier); ok {
+			notifier.AddListener(dynamicCertificateController)
+		}
 		// start controllers if possible
 		if controller, ok := s.ClientCA.(dynamiccertificates.ControllerRunner); ok {
 			// runonce to be sure that we have a value.
@@ -85,6 +88,14 @@ func (s *SecureServingInfo) tlsConfig(stopCh <-chan struct{}) (*tls.Config, erro
 
 			go controller.Run(1, stopCh)
 		}
+		if controller, ok := s.Cert.(dynamiccertificates.ControllerRunner); ok {
+			// runonce to be sure that we have a value.
+			if err := controller.RunOnce(); err != nil {
+				return nil, err
+			}
+
+			go controller.Run(1, stopCh)
+		}
 
 		// runonce to be sure that we have a value.
 		if err := dynamicCertificateController.RunOnce(); err != nil {