csr: add expirationSeconds field to control cert lifetime

This change updates the CSR API to add a new, optional field called
expirationSeconds.  This field is a request to the signer for the
maximum duration the client wishes the cert to have.  The signer is
free to ignore this request based on its own internal policy.  The
signers built-in to KCM will honor this field if it is not set to a
value greater than --cluster-signing-duration.  The minimum allowed
value for this field is 600 seconds (ten minutes).

This change will help enforce safer durations for certificates in
the Kube ecosystem and will help related projects such as
cert-manager with their migration to the Kube CSR API.

Future enhancements may update the Kubelet to take advantage of this
field when it is configured in a way that can tolerate shorter
certificate lifespans with regular rotation.

Signed-off-by: Monis Khan <mok@vmware.com>
This commit is contained in:
Monis Khan
2021-06-25 22:08:10 -04:00
parent 0dad7d1c47
commit cd91e59f7c
24 changed files with 1413 additions and 71 deletions

View File

@@ -17,9 +17,12 @@ limitations under the License.
package fuzzer
import (
"time"
fuzz "github.com/google/gofuzz"
runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/client-go/util/certificate/csr"
"k8s.io/kubernetes/pkg/apis/certificates"
api "k8s.io/kubernetes/pkg/apis/core"
)
@@ -31,6 +34,7 @@ var Funcs = func(codecs runtimeserializer.CodecFactory) []interface{} {
c.FuzzNoCustom(obj) // fuzz self without calling this function again
obj.Usages = []certificates.KeyUsage{certificates.UsageKeyEncipherment}
obj.SignerName = "example.com/custom-sample-signer"
obj.ExpirationSeconds = csr.DurationToExpirationSeconds(time.Hour + time.Minute + time.Second)
},
func(obj *certificates.CertificateSigningRequestCondition, c fuzz.Continue) {
c.FuzzNoCustom(obj) // fuzz self without calling this function again

View File

@@ -68,6 +68,30 @@ type CertificateSigningRequestSpec struct {
// 6. Whether or not requests for CA certificates are allowed.
SignerName string
// expirationSeconds is the requested duration of validity of the issued
// certificate. The certificate signer may issue a certificate with a different
// validity duration so a client must check the delta between the notBefore and
// and notAfter fields in the issued certificate to determine the actual duration.
//
// The v1.22+ in-tree implementations of the well-known Kubernetes signers will
// honor this field as long as the requested duration is not greater than the
// maximum duration they will honor per the --cluster-signing-duration CLI
// flag to the Kubernetes controller manager.
//
// Certificate signers may not honor this field for various reasons:
//
// 1. Old signer that is unaware of the field (such as the in-tree
// implementations prior to v1.22)
// 2. Signer whose configured maximum is shorter than the requested duration
// 3. Signer whose configured minimum is longer than the requested duration
//
// The minimum valid value for expirationSeconds is 600, i.e. 10 minutes.
//
// As of v1.22, this field is beta and is controlled via the CSRDuration feature gate.
//
// +optional
ExpirationSeconds *int32
// usages specifies a set of usage contexts the key will be
// valid for.
// See: https://tools.ietf.org/html/rfc5280#section-4.2.1.3

View File

@@ -205,6 +205,9 @@ func validateCertificateSigningRequest(csr *certificates.CertificateSigningReque
} else {
allErrs = append(allErrs, ValidateCertificateSigningRequestSignerName(specPath.Child("signerName"), csr.Spec.SignerName)...)
}
if csr.Spec.ExpirationSeconds != nil && *csr.Spec.ExpirationSeconds < 600 {
allErrs = append(allErrs, field.Invalid(specPath.Child("expirationSeconds"), *csr.Spec.ExpirationSeconds, "may not specify a duration less than 600 seconds (10 minutes)"))
}
allErrs = append(allErrs, validateConditions(field.NewPath("status", "conditions"), csr, opts)...)
if !opts.allowArbitraryCertificate {

View File

@@ -26,14 +26,17 @@ import (
"reflect"
"strings"
"testing"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/client-go/util/certificate/csr"
capi "k8s.io/kubernetes/pkg/apis/certificates"
capiv1beta1 "k8s.io/kubernetes/pkg/apis/certificates/v1beta1"
"k8s.io/kubernetes/pkg/apis/core"
"k8s.io/utils/pointer"
)
var (
@@ -262,6 +265,74 @@ func TestValidateCertificateSigningRequestCreate(t *testing.T) {
},
errs: field.ErrorList{},
},
"negative duration": {
csr: capi.CertificateSigningRequest{
ObjectMeta: validObjectMeta,
Spec: capi.CertificateSigningRequestSpec{
Usages: validUsages,
Request: newCSRPEM(t),
SignerName: validSignerName,
ExpirationSeconds: pointer.Int32(-1),
},
},
errs: field.ErrorList{
field.Invalid(specPath.Child("expirationSeconds"), int32(-1), "may not specify a duration less than 600 seconds (10 minutes)"),
},
},
"zero duration": {
csr: capi.CertificateSigningRequest{
ObjectMeta: validObjectMeta,
Spec: capi.CertificateSigningRequestSpec{
Usages: validUsages,
Request: newCSRPEM(t),
SignerName: validSignerName,
ExpirationSeconds: pointer.Int32(0),
},
},
errs: field.ErrorList{
field.Invalid(specPath.Child("expirationSeconds"), int32(0), "may not specify a duration less than 600 seconds (10 minutes)"),
},
},
"one duration": {
csr: capi.CertificateSigningRequest{
ObjectMeta: validObjectMeta,
Spec: capi.CertificateSigningRequestSpec{
Usages: validUsages,
Request: newCSRPEM(t),
SignerName: validSignerName,
ExpirationSeconds: pointer.Int32(1),
},
},
errs: field.ErrorList{
field.Invalid(specPath.Child("expirationSeconds"), int32(1), "may not specify a duration less than 600 seconds (10 minutes)"),
},
},
"too short duration": {
csr: capi.CertificateSigningRequest{
ObjectMeta: validObjectMeta,
Spec: capi.CertificateSigningRequestSpec{
Usages: validUsages,
Request: newCSRPEM(t),
SignerName: validSignerName,
ExpirationSeconds: csr.DurationToExpirationSeconds(time.Minute),
},
},
errs: field.ErrorList{
field.Invalid(specPath.Child("expirationSeconds"), *csr.DurationToExpirationSeconds(time.Minute), "may not specify a duration less than 600 seconds (10 minutes)"),
},
},
"valid duration": {
csr: capi.CertificateSigningRequest{
ObjectMeta: validObjectMeta,
Spec: capi.CertificateSigningRequestSpec{
Usages: validUsages,
Request: newCSRPEM(t),
SignerName: validSignerName,
ExpirationSeconds: csr.DurationToExpirationSeconds(10 * time.Minute),
},
},
errs: field.ErrorList{},
},
"missing usages": {
csr: capi.CertificateSigningRequest{
ObjectMeta: validObjectMeta,

View File

@@ -38,8 +38,8 @@ type CSRSigningControllerConfiguration struct {
// legacyUnknownSignerConfiguration holds the certificate and key used to issue certificates for the kubernetes.io/legacy-unknown
LegacyUnknownSignerConfiguration CSRSigningConfiguration
// clusterSigningDuration is the length of duration signed certificates
// will be given.
// clusterSigningDuration is the max length of duration signed certificates will be given.
// Individual CSRs may request shorter certs by setting spec.expirationSeconds.
ClusterSigningDuration metav1.Duration
}

View File

@@ -30,11 +30,14 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/server/dynamiccertificates"
utilfeature "k8s.io/apiserver/pkg/util/feature"
certificatesinformers "k8s.io/client-go/informers/certificates/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/util/certificate/csr"
capihelper "k8s.io/kubernetes/pkg/apis/certificates"
"k8s.io/kubernetes/pkg/controller/certificates"
"k8s.io/kubernetes/pkg/controller/certificates/authority"
"k8s.io/kubernetes/pkg/features"
)
type CSRSigningController struct {
@@ -115,7 +118,7 @@ type signer struct {
caProvider *caProvider
client clientset.Interface
certTTL time.Duration
certTTL time.Duration // max TTL; individual requests may request shorter certs by setting spec.expirationSeconds
signerName string
isRequestForSignerFn isRequestForSignerFunc
@@ -173,7 +176,7 @@ func (s *signer) handle(csr *capi.CertificateSigningRequest) error {
// Ignore requests for kubernetes.io signerNames we don't recognize
return nil
}
cert, err := s.sign(x509cr, csr.Spec.Usages, nil)
cert, err := s.sign(x509cr, csr.Spec.Usages, csr.Spec.ExpirationSeconds, nil)
if err != nil {
return fmt.Errorf("error auto signing csr: %v", err)
}
@@ -185,15 +188,15 @@ func (s *signer) handle(csr *capi.CertificateSigningRequest) error {
return nil
}
func (s *signer) sign(x509cr *x509.CertificateRequest, usages []capi.KeyUsage, now func() time.Time) ([]byte, error) {
func (s *signer) sign(x509cr *x509.CertificateRequest, usages []capi.KeyUsage, expirationSeconds *int32, now func() time.Time) ([]byte, error) {
currCA, err := s.caProvider.currentCA()
if err != nil {
return nil, err
}
der, err := currCA.Sign(x509cr.Raw, authority.PermissiveSigningPolicy{
TTL: s.certTTL,
TTL: s.duration(expirationSeconds),
Usages: usages,
Backdate: 5 * time.Minute, // this must always be less than the minimum TTL requested by a user
Backdate: 5 * time.Minute, // this must always be less than the minimum TTL requested by a user (see sanity check requestedDuration below)
Short: 8 * time.Hour, // 5 minutes of backdating is roughly 1% of 8 hours
Now: now,
})
@@ -203,6 +206,30 @@ func (s *signer) sign(x509cr *x509.CertificateRequest, usages []capi.KeyUsage, n
return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), nil
}
func (s *signer) duration(expirationSeconds *int32) time.Duration {
if !utilfeature.DefaultFeatureGate.Enabled(features.CSRDuration) {
return s.certTTL
}
if expirationSeconds == nil {
return s.certTTL
}
// honor requested duration is if it is less than the default TTL
// use 10 min (2x hard coded backdate above) as a sanity check lower bound
const min = 10 * time.Minute
switch requestedDuration := csr.ExpirationSecondsToDuration(*expirationSeconds); {
case requestedDuration > s.certTTL:
return s.certTTL
case requestedDuration < min:
return min
default:
return requestedDuration
}
}
// getCSRVerificationFuncForSignerName is a function that provides reliable mapping of signer names to verification so that
// we don't have accidents with wiring at some later date.
func getCSRVerificationFuncForSignerName(signerName string) (isRequestForSignerFunc, error) {

View File

@@ -32,11 +32,15 @@ import (
capi "k8s.io/api/certificates/v1"
"k8s.io/apimachinery/pkg/util/clock"
"k8s.io/apimachinery/pkg/util/diff"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/kubernetes/fake"
testclient "k8s.io/client-go/testing"
"k8s.io/client-go/util/cert"
"k8s.io/client-go/util/certificate/csr"
featuregatetesting "k8s.io/component-base/featuregate/testing"
capihelper "k8s.io/kubernetes/pkg/apis/certificates/v1"
"k8s.io/kubernetes/pkg/controller/certificates"
"k8s.io/kubernetes/pkg/features"
)
func TestSigner(t *testing.T) {
@@ -61,7 +65,11 @@ func TestSigner(t *testing.T) {
capi.UsageKeyEncipherment,
capi.UsageServerAuth,
capi.UsageClientAuth,
}, fakeClock.Now)
},
// requesting a duration that is greater than TTL is ignored
csr.DurationToExpirationSeconds(3*time.Hour),
fakeClock.Now,
)
if err != nil {
t.Fatalf("failed to sign CSR: %v", err)
}
@@ -344,3 +352,91 @@ func makeTestCSR(b csrBuilder) *capi.CertificateSigningRequest {
}
return csr
}
func Test_signer_duration(t *testing.T) {
t.Parallel()
tests := []struct {
name string
certTTL time.Duration
expirationSeconds *int32
wantGateEnabled time.Duration
wantGateDisabled time.Duration
}{
{
name: "can request shorter duration than TTL",
certTTL: time.Hour,
expirationSeconds: csr.DurationToExpirationSeconds(30 * time.Minute),
wantGateEnabled: 30 * time.Minute,
wantGateDisabled: time.Hour,
},
{
name: "cannot request longer duration than TTL",
certTTL: time.Hour,
expirationSeconds: csr.DurationToExpirationSeconds(3 * time.Hour),
wantGateEnabled: time.Hour,
wantGateDisabled: time.Hour,
},
{
name: "cannot request negative duration",
certTTL: time.Hour,
expirationSeconds: csr.DurationToExpirationSeconds(-time.Minute),
wantGateEnabled: 10 * time.Minute,
wantGateDisabled: time.Hour,
},
{
name: "cannot request duration less than 10 mins",
certTTL: time.Hour,
expirationSeconds: csr.DurationToExpirationSeconds(10*time.Minute - time.Second),
wantGateEnabled: 10 * time.Minute,
wantGateDisabled: time.Hour,
},
{
name: "can request duration of exactly 10 mins",
certTTL: time.Hour,
expirationSeconds: csr.DurationToExpirationSeconds(10 * time.Minute),
wantGateEnabled: 10 * time.Minute,
wantGateDisabled: time.Hour,
},
{
name: "can request duration equal to the default",
certTTL: time.Hour,
expirationSeconds: csr.DurationToExpirationSeconds(time.Hour),
wantGateEnabled: time.Hour,
wantGateDisabled: time.Hour,
},
{
name: "can choose not to request a duration to get the default",
certTTL: time.Hour,
expirationSeconds: nil,
wantGateEnabled: time.Hour,
wantGateDisabled: time.Hour,
},
}
for _, tt := range tests {
tt := tt
f := func(t *testing.T, want time.Duration) {
s := &signer{
certTTL: tt.certTTL,
}
if got := s.duration(tt.expirationSeconds); got != want {
t.Errorf("duration() = %v, want %v", got, want)
}
}
// regular tests
t.Run(tt.name, func(t *testing.T) {
t.Parallel() // these are safe to run in parallel but not the feature gate disabled tests
f(t, tt.wantGateEnabled)
})
// same tests with the feature gate disabled
t.Run("feature gate disabled - "+tt.name, func(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSRDuration, false)()
f(t, tt.wantGateDisabled)
})
}
}

View File

@@ -740,6 +740,12 @@ const (
//
// Enables usage of the ReadWriteOncePod PersistentVolume access mode.
ReadWriteOncePod featuregate.Feature = "ReadWriteOncePod"
// owner: @enj
// beta: v1.22
//
// Allows clients to request a duration for certificates issued via the Kubernetes CSR API.
CSRDuration featuregate.Feature = "CSRDuration"
)
func init() {
@@ -851,6 +857,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
SeccompDefault: {Default: false, PreRelease: featuregate.Alpha},
PodSecurity: {Default: false, PreRelease: featuregate.Alpha},
ReadWriteOncePod: {Default: false, PreRelease: featuregate.Alpha},
CSRDuration: {Default: true, PreRelease: featuregate.Beta},
// inherited features from generic apiserver, relisted here to get a conflict if it is changed
// unintentionally on either side:

View File

@@ -344,7 +344,7 @@ func requestNodeCertificate(ctx context.Context, client clientset.Interface, pri
return nil, err
}
reqName, reqUID, err := csr.RequestCertificate(client, csrData, name, certificatesv1.KubeAPIServerClientKubeletSignerName, usages, privateKey)
reqName, reqUID, err := csr.RequestCertificate(client, csrData, name, certificatesv1.KubeAPIServerClientKubeletSignerName, nil, usages, privateKey)
if err != nil {
return nil, err
}

View File

@@ -48,6 +48,7 @@ import (
"k8s.io/apimachinery/pkg/util/duration"
"k8s.io/apimachinery/pkg/util/sets"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/util/certificate/csr"
"k8s.io/kubernetes/pkg/apis/admissionregistration"
"k8s.io/kubernetes/pkg/apis/apiserverinternal"
"k8s.io/kubernetes/pkg/apis/apps"
@@ -413,6 +414,7 @@ func AddHandlers(h printers.PrintHandler) {
{Name: "Age", Type: "string", Description: metav1.ObjectMeta{}.SwaggerDoc()["creationTimestamp"]},
{Name: "SignerName", Type: "string", Description: certificatesv1beta1.CertificateSigningRequestSpec{}.SwaggerDoc()["signerName"]},
{Name: "Requestor", Type: "string", Description: certificatesv1beta1.CertificateSigningRequestSpec{}.SwaggerDoc()["request"]},
{Name: "RequestedDuration", Type: "string", Description: certificatesv1beta1.CertificateSigningRequestSpec{}.SwaggerDoc()["expirationSeconds"]},
{Name: "Condition", Type: "string", Description: certificatesv1beta1.CertificateSigningRequestStatus{}.SwaggerDoc()["conditions"]},
}
h.TableHandler(certificateSigningRequestColumnDefinitions, printCertificateSigningRequest)
@@ -1902,7 +1904,11 @@ func printCertificateSigningRequest(obj *certificates.CertificateSigningRequest,
if obj.Spec.SignerName != "" {
signerName = obj.Spec.SignerName
}
row.Cells = append(row.Cells, obj.Name, translateTimestampSince(obj.CreationTimestamp), signerName, obj.Spec.Username, status)
requestedDuration := "<none>"
if obj.Spec.ExpirationSeconds != nil {
requestedDuration = duration.HumanDuration(csr.ExpirationSecondsToDuration(*obj.Spec.ExpirationSeconds))
}
row.Cells = append(row.Cells, obj.Name, translateTimestampSince(obj.CreationTimestamp), signerName, obj.Spec.Username, requestedDuration, status)
return []metav1.TableRow{row}, nil
}

View File

@@ -27,6 +27,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/diff"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/client-go/util/certificate/csr"
"k8s.io/kubernetes/pkg/apis/apiserverinternal"
"k8s.io/kubernetes/pkg/apis/apps"
"k8s.io/kubernetes/pkg/apis/autoscaling"
@@ -3990,8 +3991,8 @@ func TestPrintCertificateSigningRequest(t *testing.T) {
Spec: certificates.CertificateSigningRequestSpec{},
Status: certificates.CertificateSigningRequestStatus{},
},
// Columns: Name, Age, Requestor, Condition
expected: []metav1.TableRow{{Cells: []interface{}{"csr1", "0s", "<none>", "", "Pending"}}},
// Columns: Name, Age, SignerName, Requestor, RequestedDuration, Condition
expected: []metav1.TableRow{{Cells: []interface{}{"csr1", "0s", "<none>", "", "<none>", "Pending"}}},
},
// Basic CSR with Spec and Status=Approved.
{
@@ -4011,8 +4012,8 @@ func TestPrintCertificateSigningRequest(t *testing.T) {
},
},
},
// Columns: Name, Age, Requestor, Condition
expected: []metav1.TableRow{{Cells: []interface{}{"csr2", "0s", "<none>", "CSR Requestor", "Approved"}}},
// Columns: Name, Age, SignerName, Requestor, RequestedDuration, Condition
expected: []metav1.TableRow{{Cells: []interface{}{"csr2", "0s", "<none>", "CSR Requestor", "<none>", "Approved"}}},
},
// Basic CSR with Spec and SignerName set
{
@@ -4033,8 +4034,31 @@ func TestPrintCertificateSigningRequest(t *testing.T) {
},
},
},
// Columns: Name, Age, Requestor, Condition
expected: []metav1.TableRow{{Cells: []interface{}{"csr2", "0s", "example.com/test-signer", "CSR Requestor", "Approved"}}},
// Columns: Name, Age, SignerName, Requestor, RequestedDuration, Condition
expected: []metav1.TableRow{{Cells: []interface{}{"csr2", "0s", "example.com/test-signer", "CSR Requestor", "<none>", "Approved"}}},
},
// Basic CSR with Spec, SignerName and ExpirationSeconds set
{
csr: certificates.CertificateSigningRequest{
ObjectMeta: metav1.ObjectMeta{
Name: "csr2",
CreationTimestamp: metav1.Time{Time: time.Now().Add(1.9e9)},
},
Spec: certificates.CertificateSigningRequestSpec{
Username: "CSR Requestor",
SignerName: "example.com/test-signer",
ExpirationSeconds: csr.DurationToExpirationSeconds(7*24*time.Hour + time.Hour), // a little bit more than a week
},
Status: certificates.CertificateSigningRequestStatus{
Conditions: []certificates.CertificateSigningRequestCondition{
{
Type: certificates.CertificateApproved,
},
},
},
},
// Columns: Name, Age, SignerName, Requestor, RequestedDuration, Condition
expected: []metav1.TableRow{{Cells: []interface{}{"csr2", "0s", "example.com/test-signer", "CSR Requestor", "7d1h", "Approved"}}},
},
// Basic CSR with Spec and Status=Approved; certificate issued.
{
@@ -4055,8 +4079,8 @@ func TestPrintCertificateSigningRequest(t *testing.T) {
Certificate: []byte("cert data"),
},
},
// Columns: Name, Age, Requestor, Condition
expected: []metav1.TableRow{{Cells: []interface{}{"csr2", "0s", "<none>", "CSR Requestor", "Approved,Issued"}}},
// Columns: Name, Age, SignerName, Requestor, RequestedDuration, Condition
expected: []metav1.TableRow{{Cells: []interface{}{"csr2", "0s", "<none>", "CSR Requestor", "<none>", "Approved,Issued"}}},
},
// Basic CSR with Spec and Status=Denied.
{
@@ -4076,8 +4100,8 @@ func TestPrintCertificateSigningRequest(t *testing.T) {
},
},
},
// Columns: Name, Age, Requestor, Condition
expected: []metav1.TableRow{{Cells: []interface{}{"csr3", "0s", "<none>", "CSR Requestor", "Denied"}}},
// Columns: Name, Age, SignerName, Requestor, RequestedDuration, Condition
expected: []metav1.TableRow{{Cells: []interface{}{"csr3", "0s", "<none>", "CSR Requestor", "<none>", "Denied"}}},
},
}

View File

@@ -0,0 +1,169 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package storage
import (
"context"
"fmt"
"strings"
"sync"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
"k8s.io/apiserver/pkg/util/dryrun"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/util/cert"
"k8s.io/client-go/util/certificate/csr"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
"k8s.io/kubernetes/pkg/apis/certificates"
"k8s.io/kubernetes/pkg/features"
)
const (
namespace = "apiserver"
subsystem = "certificates_registry"
)
var (
// csrDurationRequested counts and categorizes how many certificates were issued when the client requested a duration.
csrDurationRequested = metrics.NewCounterVec(
&metrics.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "csr_requested_duration_total",
Help: "Total number of issued CSRs with a requested duration, sliced by signer (only kubernetes.io signer names are specifically identified)",
StabilityLevel: metrics.ALPHA,
},
[]string{"signerName"},
)
// csrDurationHonored counts and categorizes how many certificates were issued when the client requested a duration and the signer honored it.
csrDurationHonored = metrics.NewCounterVec(
&metrics.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "csr_honored_duration_total",
Help: "Total number of issued CSRs with a requested duration that was honored, sliced by signer (only kubernetes.io signer names are specifically identified)",
StabilityLevel: metrics.ALPHA,
},
[]string{"signerName"},
)
)
func init() {
registerMetricsOnce.Do(func() {
legacyregistry.MustRegister(csrDurationRequested)
legacyregistry.MustRegister(csrDurationHonored)
})
}
var registerMetricsOnce sync.Once
type counterVecMetric interface {
WithLabelValues(...string) metrics.CounterMetric
}
func countCSRDurationMetric(requested, honored counterVecMetric) genericregistry.BeginUpdateFunc {
return func(ctx context.Context, obj, old runtime.Object, options *metav1.UpdateOptions) (genericregistry.FinishFunc, error) {
return func(ctx context.Context, success bool) {
if !success {
return // ignore failures
}
if dryrun.IsDryRun(options.DryRun) {
return // ignore things that would not get persisted
}
if !utilfeature.DefaultFeatureGate.Enabled(features.CSRDuration) {
return
}
oldCSR, ok := old.(*certificates.CertificateSigningRequest)
if !ok {
return
}
// if the old CSR already has a certificate, do not double count it
if len(oldCSR.Status.Certificate) > 0 {
return
}
if oldCSR.Spec.ExpirationSeconds == nil {
return // ignore CSRs that are not using the CSR duration feature
}
newCSR, ok := obj.(*certificates.CertificateSigningRequest)
if !ok {
return
}
issuedCert := newCSR.Status.Certificate
// new CSR has no issued certificate yet so do not count it.
// note that this means that we will ignore CSRs that set a duration
// but never get approved/signed. this is fine because the point
// of these metrics is to understand if the duration is honored
// by the signer. we are not checking the behavior of the approver.
if len(issuedCert) == 0 {
return
}
signer := compressSignerName(oldCSR.Spec.SignerName)
// at this point we know that this CSR is going to be persisted and
// the cert was just issued and the client requested a duration
requested.WithLabelValues(signer).Inc()
certs, err := cert.ParseCertsPEM(issuedCert)
if err != nil {
utilruntime.HandleError(fmt.Errorf("metrics recording failed to parse certificate for CSR %s: %w", oldCSR.Name, err))
return
}
// now we check to see if the signer honored the requested duration
certificate := certs[0]
wantDuration := csr.ExpirationSecondsToDuration(*oldCSR.Spec.ExpirationSeconds)
actualDuration := certificate.NotAfter.Sub(certificate.NotBefore)
if isDurationHonored(wantDuration, actualDuration) {
honored.WithLabelValues(signer).Inc()
}
}, nil
}
}
func isDurationHonored(want, got time.Duration) bool {
delta := want - got
if delta < 0 {
delta = -delta
}
// short-lived cert backdating + 5% of want
maxDelta := 5*time.Minute + (want / 20)
return delta < maxDelta
}
func compressSignerName(name string) string {
if strings.HasPrefix(name, "kubernetes.io/") {
return name
}
return "other"
}

View File

@@ -0,0 +1,496 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package storage
import (
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"math/big"
"testing"
"time"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
utilfeature "k8s.io/apiserver/pkg/util/feature"
certutil "k8s.io/client-go/util/cert"
"k8s.io/client-go/util/certificate/csr"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/component-base/metrics"
"k8s.io/kubernetes/pkg/apis/certificates"
"k8s.io/kubernetes/pkg/features"
"k8s.io/utils/pointer"
)
func Test_countCSRDurationMetric(t *testing.T) {
caPrivateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
caCert, err := certutil.NewSelfSignedCACert(certutil.Config{CommonName: "test-ca"}, caPrivateKey)
if err != nil {
t.Fatal(err)
}
tests := []struct {
name string
disableFeatureGate bool
success bool
obj, old runtime.Object
options *metav1.UpdateOptions
wantSigner string
wantRequested, wantHonored bool
}{
{
name: "cert parse failure",
success: true,
obj: &certificates.CertificateSigningRequest{
Status: certificates.CertificateSigningRequestStatus{
Certificate: []byte("junk"),
},
},
old: &certificates.CertificateSigningRequest{
Spec: certificates.CertificateSigningRequestSpec{
SignerName: "fancy",
ExpirationSeconds: pointer.Int32(77),
},
},
options: &metav1.UpdateOptions{},
wantSigner: "other",
wantRequested: true,
wantHonored: false,
},
{
name: "kube signer honors duration exactly",
success: true,
obj: &certificates.CertificateSigningRequest{
Status: certificates.CertificateSigningRequestStatus{
Certificate: createCert(t, time.Hour, caPrivateKey, caCert),
},
},
old: &certificates.CertificateSigningRequest{
Spec: certificates.CertificateSigningRequestSpec{
SignerName: "kubernetes.io/educate-dolphins",
ExpirationSeconds: csr.DurationToExpirationSeconds(time.Hour),
},
},
options: &metav1.UpdateOptions{},
wantSigner: "kubernetes.io/educate-dolphins",
wantRequested: true,
wantHonored: true,
},
{
name: "signer honors duration exactly",
success: true,
obj: &certificates.CertificateSigningRequest{
Status: certificates.CertificateSigningRequestStatus{
Certificate: createCert(t, time.Hour, caPrivateKey, caCert),
},
},
old: &certificates.CertificateSigningRequest{
Spec: certificates.CertificateSigningRequestSpec{
SignerName: "pandas",
ExpirationSeconds: csr.DurationToExpirationSeconds(time.Hour),
},
},
options: &metav1.UpdateOptions{},
wantSigner: "other",
wantRequested: true,
wantHonored: true,
},
{
name: "signer honors duration but just a little bit less",
success: true,
obj: &certificates.CertificateSigningRequest{
Status: certificates.CertificateSigningRequestStatus{
Certificate: createCert(t, time.Hour-6*time.Minute, caPrivateKey, caCert),
},
},
old: &certificates.CertificateSigningRequest{
Spec: certificates.CertificateSigningRequestSpec{
SignerName: "pandas",
ExpirationSeconds: csr.DurationToExpirationSeconds(time.Hour),
},
},
options: &metav1.UpdateOptions{},
wantSigner: "other",
wantRequested: true,
wantHonored: true,
},
{
name: "signer honors duration but just a little bit more",
success: true,
obj: &certificates.CertificateSigningRequest{
Status: certificates.CertificateSigningRequestStatus{
Certificate: createCert(t, time.Hour+6*time.Minute, caPrivateKey, caCert),
},
},
old: &certificates.CertificateSigningRequest{
Spec: certificates.CertificateSigningRequestSpec{
SignerName: "pandas",
ExpirationSeconds: csr.DurationToExpirationSeconds(time.Hour),
},
},
options: &metav1.UpdateOptions{},
wantSigner: "other",
wantRequested: true,
wantHonored: true,
},
{
name: "honors duration lower bound",
success: true,
obj: &certificates.CertificateSigningRequest{
Status: certificates.CertificateSigningRequestStatus{
Certificate: createCert(t, 651*time.Second, caPrivateKey, caCert),
},
},
old: &certificates.CertificateSigningRequest{
Spec: certificates.CertificateSigningRequestSpec{
SignerName: "kubernetes.io/educate-dolphins",
ExpirationSeconds: csr.DurationToExpirationSeconds(1_000 * time.Second),
},
},
options: &metav1.UpdateOptions{},
wantSigner: "kubernetes.io/educate-dolphins",
wantRequested: true,
wantHonored: true,
},
{
name: "does not honor duration just outside of lower bound",
success: true,
obj: &certificates.CertificateSigningRequest{
Status: certificates.CertificateSigningRequestStatus{
Certificate: createCert(t, 650*time.Second, caPrivateKey, caCert),
},
},
old: &certificates.CertificateSigningRequest{
Spec: certificates.CertificateSigningRequestSpec{
SignerName: "kubernetes.io/educate-dolphins",
ExpirationSeconds: csr.DurationToExpirationSeconds(1_000 * time.Second),
},
},
options: &metav1.UpdateOptions{},
wantSigner: "kubernetes.io/educate-dolphins",
wantRequested: true,
wantHonored: false,
},
{
name: "honors duration upper bound",
success: true,
obj: &certificates.CertificateSigningRequest{
Status: certificates.CertificateSigningRequestStatus{
Certificate: createCert(t, 1349*time.Second, caPrivateKey, caCert),
},
},
old: &certificates.CertificateSigningRequest{
Spec: certificates.CertificateSigningRequestSpec{
SignerName: "kubernetes.io/educate-dolphins",
ExpirationSeconds: csr.DurationToExpirationSeconds(1_000 * time.Second),
},
},
options: &metav1.UpdateOptions{},
wantSigner: "kubernetes.io/educate-dolphins",
wantRequested: true,
wantHonored: true,
},
{
name: "does not honor duration just outside of upper bound",
success: true,
obj: &certificates.CertificateSigningRequest{
Status: certificates.CertificateSigningRequestStatus{
Certificate: createCert(t, 1350*time.Second, caPrivateKey, caCert),
},
},
old: &certificates.CertificateSigningRequest{
Spec: certificates.CertificateSigningRequestSpec{
SignerName: "kubernetes.io/educate-dolphins",
ExpirationSeconds: csr.DurationToExpirationSeconds(1_000 * time.Second),
},
},
options: &metav1.UpdateOptions{},
wantSigner: "kubernetes.io/educate-dolphins",
wantRequested: true,
wantHonored: false,
},
{
name: "failed update is ignored",
success: false,
obj: &certificates.CertificateSigningRequest{
Status: certificates.CertificateSigningRequestStatus{
Certificate: createCert(t, time.Hour, caPrivateKey, caCert),
},
},
old: &certificates.CertificateSigningRequest{
Spec: certificates.CertificateSigningRequestSpec{
SignerName: "pandas",
ExpirationSeconds: csr.DurationToExpirationSeconds(time.Hour),
},
},
options: &metav1.UpdateOptions{},
wantSigner: "",
wantRequested: false,
wantHonored: false,
},
{
name: "dry run is ignored",
success: true,
obj: &certificates.CertificateSigningRequest{
Status: certificates.CertificateSigningRequestStatus{
Certificate: createCert(t, time.Hour, caPrivateKey, caCert),
},
},
old: &certificates.CertificateSigningRequest{
Spec: certificates.CertificateSigningRequestSpec{
SignerName: "pandas",
ExpirationSeconds: csr.DurationToExpirationSeconds(time.Hour),
},
},
options: &metav1.UpdateOptions{DryRun: []string{"stuff"}},
wantSigner: "",
wantRequested: false,
wantHonored: false,
},
{
name: "no metrics when feature gate is turned off",
disableFeatureGate: true,
success: true,
obj: &certificates.CertificateSigningRequest{
Status: certificates.CertificateSigningRequestStatus{
Certificate: createCert(t, time.Hour, caPrivateKey, caCert),
},
},
old: &certificates.CertificateSigningRequest{
Spec: certificates.CertificateSigningRequestSpec{
SignerName: "pandas",
ExpirationSeconds: csr.DurationToExpirationSeconds(time.Hour),
},
},
options: &metav1.UpdateOptions{},
wantSigner: "",
wantRequested: false,
wantHonored: false,
},
{
name: "old CSR already has a cert so it is ignored",
success: true,
obj: &certificates.CertificateSigningRequest{
Status: certificates.CertificateSigningRequestStatus{
Certificate: createCert(t, time.Hour, caPrivateKey, caCert),
},
},
old: &certificates.CertificateSigningRequest{
Spec: certificates.CertificateSigningRequestSpec{
SignerName: "pandas",
ExpirationSeconds: csr.DurationToExpirationSeconds(time.Hour),
},
Status: certificates.CertificateSigningRequestStatus{
Certificate: []byte("junk"),
},
},
options: &metav1.UpdateOptions{},
wantSigner: "",
wantRequested: false,
wantHonored: false,
},
{
name: "CSRs with no duration are ignored",
success: true,
obj: &certificates.CertificateSigningRequest{
Status: certificates.CertificateSigningRequestStatus{
Certificate: createCert(t, time.Hour, caPrivateKey, caCert),
},
},
old: &certificates.CertificateSigningRequest{
Spec: certificates.CertificateSigningRequestSpec{
SignerName: "pandas",
ExpirationSeconds: nil,
},
},
options: &metav1.UpdateOptions{},
wantSigner: "",
wantRequested: false,
wantHonored: false,
},
{
name: "unissued CSRs are ignored",
success: true,
obj: &certificates.CertificateSigningRequest{
Status: certificates.CertificateSigningRequestStatus{
Certificate: nil,
},
},
old: &certificates.CertificateSigningRequest{
Spec: certificates.CertificateSigningRequestSpec{
SignerName: "pandas",
ExpirationSeconds: csr.DurationToExpirationSeconds(time.Hour),
},
},
options: &metav1.UpdateOptions{},
wantSigner: "",
wantRequested: false,
wantHonored: false,
},
{
name: "invalid data - nil old object",
success: true,
obj: &certificates.CertificateSigningRequest{
Status: certificates.CertificateSigningRequestStatus{
Certificate: createCert(t, time.Hour, caPrivateKey, caCert),
},
},
old: nil,
options: &metav1.UpdateOptions{},
wantSigner: "",
wantRequested: false,
wantHonored: false,
},
{
name: "invalid data - nil new object",
success: true,
obj: nil,
old: &certificates.CertificateSigningRequest{
Spec: certificates.CertificateSigningRequestSpec{
SignerName: "pandas",
ExpirationSeconds: csr.DurationToExpirationSeconds(time.Hour),
},
},
options: &metav1.UpdateOptions{},
wantSigner: "",
wantRequested: false,
wantHonored: false,
},
{
name: "invalid data - junk old object",
success: true,
obj: &certificates.CertificateSigningRequest{
Status: certificates.CertificateSigningRequestStatus{
Certificate: createCert(t, time.Hour, caPrivateKey, caCert),
},
},
old: &corev1.Pod{},
options: &metav1.UpdateOptions{},
wantSigner: "",
wantRequested: false,
wantHonored: false,
},
{
name: "invalid data - junk new object",
success: true,
obj: &corev1.Pod{},
old: &certificates.CertificateSigningRequest{
Spec: certificates.CertificateSigningRequestSpec{
SignerName: "pandas",
ExpirationSeconds: csr.DurationToExpirationSeconds(time.Hour),
},
},
options: &metav1.UpdateOptions{},
wantSigner: "",
wantRequested: false,
wantHonored: false,
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
if tt.disableFeatureGate {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSRDuration, false)()
} else {
t.Parallel()
}
testReq := &testCounterVecMetric{}
testHon := &testCounterVecMetric{}
finishFunc, err := countCSRDurationMetric(testReq, testHon)(nil, tt.obj, tt.old, tt.options)
if err != nil {
t.Fatal(err)
}
finishFunc(nil, tt.success)
if got := testReq.signer; tt.wantRequested && tt.wantSigner != got {
t.Errorf("requested signer: want %v, got %v", tt.wantSigner, got)
}
if got := testHon.signer; tt.wantHonored && tt.wantSigner != got {
t.Errorf("honored signer: want %v, got %v", tt.wantSigner, got)
}
if got := testReq.called; tt.wantRequested != got {
t.Errorf("requested inc: want %v, got %v", tt.wantRequested, got)
}
if got := testHon.called; tt.wantHonored != got {
t.Errorf("honored inc: want %v, got %v", tt.wantHonored, got)
}
})
}
}
func createCert(t *testing.T, duration time.Duration, caPrivateKey *ecdsa.PrivateKey, caCert *x509.Certificate) []byte {
t.Helper()
crPublicKey := &caPrivateKey.PublicKey // this is supposed to be public key of the signee but it does not matter for this test
now := time.Now()
tmpl := &x509.Certificate{Subject: pkix.Name{CommonName: "panda"}, SerialNumber: big.NewInt(1234), NotBefore: now, NotAfter: now.Add(duration)}
der, err := x509.CreateCertificate(rand.Reader, tmpl, caCert, crPublicKey, caPrivateKey)
if err != nil {
t.Fatal(err)
}
return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
}
type testCounterVecMetric struct {
metrics.CounterMetric
signer string
called bool
}
func (m *testCounterVecMetric) WithLabelValues(lv ...string) metrics.CounterMetric {
if len(lv) != 1 {
panic(lv)
}
if len(m.signer) != 0 {
panic("unexpected multiple WithLabelValues() calls")
}
signer := lv[0]
if len(signer) == 0 {
panic("invalid empty signer")
}
m.signer = signer
return m
}
func (m *testCounterVecMetric) Inc() {
if m.called {
panic("unexpected multiple Inc() calls")
}
m.called = true
}

View File

@@ -62,6 +62,7 @@ func NewREST(optsGetter generic.RESTOptionsGetter) (*REST, *StatusREST, *Approva
statusStore := *store
statusStore.UpdateStrategy = csrregistry.StatusStrategy
statusStore.ResetFieldsStrategy = csrregistry.StatusStrategy
statusStore.BeginUpdate = countCSRDurationMetric(csrDurationRequested, csrDurationHonored)
approvalStore := *store
approvalStore.UpdateStrategy = csrregistry.ApprovalStrategy

View File

@@ -30,9 +30,11 @@ import (
genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/registry/generic"
"k8s.io/apiserver/pkg/storage/names"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/kubernetes/pkg/api/legacyscheme"
"k8s.io/kubernetes/pkg/apis/certificates"
"k8s.io/kubernetes/pkg/apis/certificates/validation"
"k8s.io/kubernetes/pkg/features"
"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
)
@@ -91,10 +93,14 @@ func (csrStrategy) PrepareForCreate(ctx context.Context, obj runtime.Object) {
if extra := user.GetExtra(); len(extra) > 0 {
csr.Spec.Extra = map[string]certificates.ExtraValue{}
for k, v := range extra {
csr.Spec.Extra[k] = certificates.ExtraValue(v)
csr.Spec.Extra[k] = v
}
}
}
// clear expirationSeconds if the CSRDuration feature is disabled
if !utilfeature.DefaultFeatureGate.Enabled(features.CSRDuration) {
csr.Spec.ExpirationSeconds = nil
}
// Be explicit that users cannot create pre-approved certificate requests.
csr.Status = certificates.CertificateSigningRequestStatus{}

View File

@@ -27,14 +27,19 @@ import (
"k8s.io/apimachinery/pkg/util/diff"
"k8s.io/apiserver/pkg/authentication/user"
genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
certapi "k8s.io/kubernetes/pkg/apis/certificates"
"k8s.io/kubernetes/pkg/features"
"k8s.io/utils/pointer"
)
func TestStrategyCreate(t *testing.T) {
tests := map[string]struct {
ctx context.Context
obj runtime.Object
expectedObj runtime.Object
ctx context.Context
disableFeatureGate bool
obj runtime.Object
expectedObj runtime.Object
}{
"no user in context, no user in obj": {
ctx: genericapirequest.NewContext(),
@@ -112,15 +117,51 @@ func TestStrategyCreate(t *testing.T) {
},
expectedObj: &certapi.CertificateSigningRequest{
Status: certapi.CertificateSigningRequestStatus{Conditions: []certapi.CertificateSigningRequestCondition{}},
}},
},
},
"expirationSeconds set with gate enabled": {
ctx: genericapirequest.NewContext(),
obj: &certapi.CertificateSigningRequest{
Spec: certapi.CertificateSigningRequestSpec{
ExpirationSeconds: pointer.Int32(1234),
},
},
expectedObj: &certapi.CertificateSigningRequest{
Spec: certapi.CertificateSigningRequestSpec{
ExpirationSeconds: pointer.Int32(1234),
},
Status: certapi.CertificateSigningRequestStatus{Conditions: []certapi.CertificateSigningRequestCondition{}},
},
},
"expirationSeconds set with gate disabled": {
ctx: genericapirequest.NewContext(),
disableFeatureGate: true,
obj: &certapi.CertificateSigningRequest{
Spec: certapi.CertificateSigningRequestSpec{
ExpirationSeconds: pointer.Int32(5678),
},
},
expectedObj: &certapi.CertificateSigningRequest{
Spec: certapi.CertificateSigningRequestSpec{
ExpirationSeconds: nil,
},
Status: certapi.CertificateSigningRequestStatus{Conditions: []certapi.CertificateSigningRequestCondition{}},
},
},
}
for k, tc := range tests {
obj := tc.obj
Strategy.PrepareForCreate(tc.ctx, obj)
if !reflect.DeepEqual(obj, tc.expectedObj) {
t.Errorf("%s: object diff: %s", k, diff.ObjectDiff(obj, tc.expectedObj))
}
tc := tc
t.Run(k, func(t *testing.T) {
if tc.disableFeatureGate {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSRDuration, false)()
}
obj := tc.obj
Strategy.PrepareForCreate(tc.ctx, obj)
if !reflect.DeepEqual(obj, tc.expectedObj) {
t.Errorf("object diff: %s", diff.ObjectDiff(obj, tc.expectedObj))
}
})
}
}