Merge pull request #118828 from enj/enj/f/kms_v2_hkdf_expand

kmsv2: KDF based nonce extension
This commit is contained in:
Kubernetes Prow Robot
2023-07-21 16:10:19 -07:00
committed by GitHub
19 changed files with 1688 additions and 374 deletions

View File

@@ -125,6 +125,13 @@ const (
// Enables KMS v2 API for encryption at rest.
KMSv2 featuregate.Feature = "KMSv2"
// owner: @enj
// kep: https://kep.k8s.io/3299
// beta: v1.28
//
// Enables the use of derived encryption keys with KMS v2.
KMSv2KDF featuregate.Feature = "KMSv2KDF"
// owner: @jiahuif
// kep: https://kep.k8s.io/2887
// alpha: v1.23
@@ -251,6 +258,8 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
KMSv2: {Default: true, PreRelease: featuregate.Beta},
KMSv2KDF: {Default: false, PreRelease: featuregate.Beta}, // default and lock to true in 1.29, remove in 1.31
OpenAPIEnums: {Default: true, PreRelease: featuregate.Beta},
OpenAPIV3: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.29

View File

@@ -47,6 +47,7 @@ import (
aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
"k8s.io/apiserver/pkg/storage/value/encrypt/envelope"
envelopekmsv2 "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2"
kmstypes "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2"
"k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics"
"k8s.io/apiserver/pkg/storage/value/encrypt/identity"
"k8s.io/apiserver/pkg/storage/value/encrypt/secretbox"
@@ -63,13 +64,13 @@ const (
kmsTransformerPrefixV2 = "k8s:enc:kms:v2:"
// these constants relate to how the KMS v2 plugin status poll logic
// and the DEK generation logic behave. In particular, the positive
// and the DEK/seed generation logic behave. In particular, the positive
// interval and max TTL are closely related as the difference between
// these values defines the worst case window in which the write DEK
// these values defines the worst case window in which the write DEK/seed
// could expire due to the plugin going into an error state. The
// worst case window divided by the negative interval defines the
// minimum amount of times the server will attempt to return to a
// healthy state before the DEK expires and writes begin to fail.
// healthy state before the DEK/seed expires and writes begin to fail.
//
// For now, these values are kept small and hardcoded to support being
// able to perform a "passive" storage migration while tolerating some
@@ -82,13 +83,13 @@ const (
// At that point, they are guaranteed to either migrate to the new key
// or get errors during the migration.
//
// If the API server coasted forever on the last DEK, they would need
// If the API server coasted forever on the last DEK/seed, they would need
// to actively check if it had observed the new key ID before starting
// a migration - otherwise it could keep using the old DEK and their
// a migration - otherwise it could keep using the old DEK/seed and their
// storage migration would not do what they thought it did.
kmsv2PluginHealthzPositiveInterval = 1 * time.Minute
kmsv2PluginHealthzNegativeInterval = 10 * time.Second
kmsv2PluginWriteDEKMaxTTL = 3 * time.Minute
kmsv2PluginWriteDEKSourceMaxTTL = 3 * time.Minute
kmsPluginHealthzNegativeTTL = 3 * time.Second
kmsPluginHealthzPositiveTTL = 20 * time.Second
@@ -332,8 +333,8 @@ func (h *kmsv2PluginProbe) check(ctx context.Context) error {
return nil
}
// rotateDEKOnKeyIDChange tries to rotate to a new DEK if the key ID returned by Status does not match the
// current state. If a successful rotation is performed, the new DEK and keyID overwrite the existing state.
// rotateDEKOnKeyIDChange tries to rotate to a new DEK/seed if the key ID returned by Status does not match the
// current state. If a successful rotation is performed, the new DEK/seed and keyID overwrite the existing state.
// On any failure during rotation (including mismatch between status and encrypt calls), the current state is
// preserved and will remain valid to use for encryption until its expiration (the system attempts to coast).
// If the key ID returned by Status matches the current state, the expiration of the current state is extended
@@ -346,32 +347,38 @@ func (h *kmsv2PluginProbe) rotateDEKOnKeyIDChange(ctx context.Context, statusKey
// allow reads indefinitely in all cases
// allow writes indefinitely as long as there is no error
// allow writes for only up to kmsv2PluginWriteDEKMaxTTL from now when there are errors
// we start the timer before we make the network call because kmsv2PluginWriteDEKMaxTTL is meant to be the upper bound
expirationTimestamp := envelopekmsv2.NowFunc().Add(kmsv2PluginWriteDEKMaxTTL)
// allow writes for only up to kmsv2PluginWriteDEKSourceMaxTTL from now when there are errors
// we start the timer before we make the network call because kmsv2PluginWriteDEKSourceMaxTTL is meant to be the upper bound
expirationTimestamp := envelopekmsv2.NowFunc().Add(kmsv2PluginWriteDEKSourceMaxTTL)
// state is valid and status keyID is unchanged from when we generated this DEK so there is no need to rotate it
// dynamically check if we want to use KDF seed to derive DEKs or just a single DEK
// this gate can only change during tests, but the check is cheap enough to always make
// this allows us to easily exercise both modes without restarting the API server
// TODO integration test that this dynamically takes effect
useSeed := utilfeature.DefaultFeatureGate.Enabled(features.KMSv2KDF)
stateUseSeed := state.EncryptedObject.EncryptedDEKSourceType == kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED
// state is valid and status keyID is unchanged from when we generated this DEK/seed so there is no need to rotate it
// just move the expiration of the current state forward by the reuse interval
if errState == nil && state.KeyID == statusKeyID {
// useSeed can only change at runtime during tests, so we check it here to allow us to easily exercise both modes
if errState == nil && state.EncryptedObject.KeyID == statusKeyID && stateUseSeed == useSeed {
state.ExpirationTimestamp = expirationTimestamp
h.state.Store(&state)
return nil
}
transformer, resp, cacheKey, errGen := envelopekmsv2.GenerateTransformer(ctx, uid, h.service)
transformer, encObject, cacheKey, errGen := envelopekmsv2.GenerateTransformer(ctx, uid, h.service, useSeed)
if resp == nil {
resp = &kmsservice.EncryptResponse{} // avoid nil panics
if encObject == nil {
encObject = &kmstypes.EncryptedObject{} // avoid nil panics
}
// happy path, should be the common case
// TODO maybe add success metrics?
if errGen == nil && resp.KeyID == statusKeyID {
if errGen == nil && encObject.KeyID == statusKeyID {
h.state.Store(&envelopekmsv2.State{
Transformer: transformer,
EncryptedDEK: resp.Ciphertext,
KeyID: resp.KeyID,
Annotations: resp.Annotations,
EncryptedObject: *encObject,
UID: uid,
ExpirationTimestamp: expirationTimestamp,
CacheKey: cacheKey,
@@ -384,8 +391,9 @@ func (h *kmsv2PluginProbe) rotateDEKOnKeyIDChange(ctx context.Context, statusKey
if klogV6.Enabled() {
klogV6.InfoS("successfully rotated DEK",
"uid", uid,
"newKeyIDHash", envelopekmsv2.GetHashIfNotEmpty(resp.KeyID),
"oldKeyIDHash", envelopekmsv2.GetHashIfNotEmpty(state.KeyID),
"useSeed", useSeed,
"newKeyIDHash", envelopekmsv2.GetHashIfNotEmpty(encObject.KeyID),
"oldKeyIDHash", envelopekmsv2.GetHashIfNotEmpty(state.EncryptedObject.KeyID),
"expirationTimestamp", expirationTimestamp.Format(time.RFC3339),
)
}
@@ -393,8 +401,8 @@ func (h *kmsv2PluginProbe) rotateDEKOnKeyIDChange(ctx context.Context, statusKey
}
}
return fmt.Errorf("failed to rotate DEK uid=%q, errState=%v, errGen=%v, statusKeyIDHash=%q, encryptKeyIDHash=%q, stateKeyIDHash=%q, expirationTimestamp=%s",
uid, errState, errGen, envelopekmsv2.GetHashIfNotEmpty(statusKeyID), envelopekmsv2.GetHashIfNotEmpty(resp.KeyID), envelopekmsv2.GetHashIfNotEmpty(state.KeyID), state.ExpirationTimestamp.Format(time.RFC3339))
return fmt.Errorf("failed to rotate DEK uid=%q, useSeed=%v, errState=%v, errGen=%v, statusKeyIDHash=%q, encryptKeyIDHash=%q, stateKeyIDHash=%q, expirationTimestamp=%s",
uid, useSeed, errState, errGen, envelopekmsv2.GetHashIfNotEmpty(statusKeyID), envelopekmsv2.GetHashIfNotEmpty(encObject.KeyID), envelopekmsv2.GetHashIfNotEmpty(state.EncryptedObject.KeyID), state.ExpirationTimestamp.Format(time.RFC3339))
}
// getCurrentState returns the latest state from the last status and encrypt calls.
@@ -407,12 +415,13 @@ func (h *kmsv2PluginProbe) getCurrentState() (envelopekmsv2.State, error) {
return envelopekmsv2.State{}, fmt.Errorf("got unexpected nil transformer")
}
if len(state.EncryptedDEK) == 0 {
return envelopekmsv2.State{}, fmt.Errorf("got unexpected empty EncryptedDEK")
encryptedObjectCopy := state.EncryptedObject
if len(encryptedObjectCopy.EncryptedData) != 0 {
return envelopekmsv2.State{}, fmt.Errorf("got unexpected non-empty EncryptedData")
}
if len(state.KeyID) == 0 {
return envelopekmsv2.State{}, fmt.Errorf("got unexpected empty keyID")
encryptedObjectCopy.EncryptedData = []byte{0} // any non-empty value to pass validation
if err := envelopekmsv2.ValidateEncryptedObject(&encryptedObjectCopy); err != nil {
return envelopekmsv2.State{}, fmt.Errorf("got invalid EncryptedObject: %w", err)
}
if state.ExpirationTimestamp.IsZero() {
@@ -772,7 +781,7 @@ func primeAndProbeKMSv2(ctx context.Context, probe *kmsv2PluginProbe, kmsName st
// make sure that the plugin's key ID is reasonably up-to-date
// also, make sure that our DEK is up-to-date to with said key ID (if it expires the server will fail all writes)
// if this background loop ever stops running, the server will become unfunctional after kmsv2PluginWriteDEKMaxTTL
// if this background loop ever stops running, the server will become unfunctional after kmsv2PluginWriteDEKSourceMaxTTL
go wait.PollUntilWithContext(
ctx,
kmsv2PluginHealthzPositiveInterval,

View File

@@ -39,6 +39,7 @@ import (
"k8s.io/apiserver/pkg/storage/value"
"k8s.io/apiserver/pkg/storage/value/encrypt/envelope"
envelopekmsv2 "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2"
kmstypes "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2"
"k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
@@ -606,7 +607,7 @@ func TestKMSPluginHealthz(t *testing.T) {
ttl: 3 * time.Second,
}
keyID := "1"
kmsv2Probe.state.Store(&envelopekmsv2.State{KeyID: keyID})
kmsv2Probe.state.Store(&envelopekmsv2.State{EncryptedObject: kmstypes.EncryptedObject{KeyID: keyID}})
testCases := []struct {
desc string
@@ -1711,6 +1712,7 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
name string
service *testKMSv2EnvelopeService
state envelopekmsv2.State
useSeed bool
statusKeyID string
wantState envelopekmsv2.State
wantEncryptCalls int
@@ -1723,13 +1725,13 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
state: envelopekmsv2.State{},
statusKeyID: "1",
wantState: envelopekmsv2.State{
KeyID: "1",
EncryptedObject: kmstypes.EncryptedObject{KeyID: "1"},
ExpirationTimestamp: now.Add(3 * time.Minute),
},
wantEncryptCalls: 1,
wantLogs: []string{
`"encrypting content using envelope service" uid="panda"`,
fmt.Sprintf(`"successfully rotated DEK" uid="panda" newKeyIDHash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b" oldKeyIDHash="" expirationTimestamp="%s"`,
fmt.Sprintf(`"successfully rotated DEK" uid="panda" useSeed=false newKeyIDHash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b" oldKeyIDHash="" expirationTimestamp="%s"`,
now.Add(3*time.Minute).Format(time.RFC3339)),
},
wantErr: "",
@@ -1740,20 +1742,38 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
state: validState(t, "2", now),
statusKeyID: "2",
wantState: envelopekmsv2.State{
KeyID: "2",
EncryptedObject: kmstypes.EncryptedObject{KeyID: "2"},
ExpirationTimestamp: now.Add(3 * time.Minute),
},
wantEncryptCalls: 0,
wantLogs: nil,
wantErr: "",
},
{
name: "happy path, with previous state, useSeed=true",
service: &testKMSv2EnvelopeService{keyID: "2"},
state: validState(t, "2", now),
useSeed: true,
statusKeyID: "2",
wantState: envelopekmsv2.State{
EncryptedObject: kmstypes.EncryptedObject{KeyID: "2", EncryptedDEKSourceType: kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED},
ExpirationTimestamp: now.Add(3 * time.Minute),
},
wantEncryptCalls: 1,
wantLogs: []string{
`"encrypting content using envelope service" uid="panda"`,
fmt.Sprintf(`"successfully rotated DEK" uid="panda" useSeed=true newKeyIDHash="sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35" oldKeyIDHash="sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35" expirationTimestamp="%s"`,
now.Add(3*time.Minute).Format(time.RFC3339)),
},
wantErr: "",
},
{
name: "previous state expired but key ID matches",
service: &testKMSv2EnvelopeService{err: fmt.Errorf("broken")}, // not called
state: validState(t, "3", now.Add(-time.Hour)),
statusKeyID: "3",
wantState: envelopekmsv2.State{
KeyID: "3",
EncryptedObject: kmstypes.EncryptedObject{KeyID: "3"},
ExpirationTimestamp: now.Add(3 * time.Minute),
},
wantEncryptCalls: 0,
@@ -1766,13 +1786,13 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
state: validState(t, "3", now.Add(-time.Hour)),
statusKeyID: "4",
wantState: envelopekmsv2.State{
KeyID: "4",
EncryptedObject: kmstypes.EncryptedObject{KeyID: "4"},
ExpirationTimestamp: now.Add(3 * time.Minute),
},
wantEncryptCalls: 1,
wantLogs: []string{
`"encrypting content using envelope service" uid="panda"`,
fmt.Sprintf(`"successfully rotated DEK" uid="panda" newKeyIDHash="sha256:4b227777d4dd1fc61c6f884f48641d02b4d121d3fd328cb08b5531fcacdabf8a" oldKeyIDHash="sha256:4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce" expirationTimestamp="%s"`,
fmt.Sprintf(`"successfully rotated DEK" uid="panda" useSeed=false newKeyIDHash="sha256:4b227777d4dd1fc61c6f884f48641d02b4d121d3fd328cb08b5531fcacdabf8a" oldKeyIDHash="sha256:4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce" expirationTimestamp="%s"`,
now.Add(3*time.Minute).Format(time.RFC3339)),
},
wantErr: "",
@@ -1783,14 +1803,14 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
state: validState(t, "4", now.Add(7*time.Minute)),
statusKeyID: "5",
wantState: envelopekmsv2.State{
KeyID: "4",
EncryptedObject: kmstypes.EncryptedObject{KeyID: "4"},
ExpirationTimestamp: now.Add(7 * time.Minute),
},
wantEncryptCalls: 1,
wantLogs: []string{
`"encrypting content using envelope service" uid="panda"`,
},
wantErr: `failed to rotate DEK uid="panda", ` +
wantErr: `failed to rotate DEK uid="panda", useSeed=false, ` +
`errState=<nil>, errGen=failed to encrypt DEK, error: broken, statusKeyIDHash="sha256:ef2d127de37b942baad06145e54b0c619a1f22327b2ebbcfbec78f5564afe39d", ` +
`encryptKeyIDHash="", stateKeyIDHash="sha256:4b227777d4dd1fc61c6f884f48641d02b4d121d3fd328cb08b5531fcacdabf8a", expirationTimestamp=` + now.Add(7*time.Minute).Format(time.RFC3339),
},
@@ -1804,7 +1824,7 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
wantLogs: []string{
`"encrypting content using envelope service" uid="panda"`,
},
wantErr: `failed to rotate DEK uid="panda", ` +
wantErr: `failed to rotate DEK uid="panda", useSeed=false, ` +
`errState=got unexpected nil transformer, errGen=failed to validate annotations: annotations: Invalid value: "panda": ` +
`should be a domain with at least two segments separated by dots, statusKeyIDHash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b", ` +
`encryptKeyIDHash="", stateKeyIDHash="", expirationTimestamp=` + (time.Time{}).Format(time.RFC3339),
@@ -1815,14 +1835,14 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
state: validState(t, "2", now),
statusKeyID: "3",
wantState: envelopekmsv2.State{
KeyID: "2",
EncryptedObject: kmstypes.EncryptedObject{KeyID: "2"},
ExpirationTimestamp: now,
},
wantEncryptCalls: 1,
wantLogs: []string{
`"encrypting content using envelope service" uid="panda"`,
},
wantErr: `failed to rotate DEK uid="panda", ` +
wantErr: `failed to rotate DEK uid="panda", useSeed=false, ` +
`errState=<nil>, errGen=failed to validate annotations: annotations: Invalid value: "panda": ` +
`should be a domain with at least two segments separated by dots, statusKeyIDHash="sha256:4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce", ` +
`encryptKeyIDHash="", stateKeyIDHash="sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35", expirationTimestamp=` + now.Format(time.RFC3339),
@@ -1830,6 +1850,8 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, tt.useSeed)()
var buf bytes.Buffer
klog.SetOutput(&buf)
@@ -1850,14 +1872,29 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
t.Errorf("log mismatch (-want +got):\n%s", diff)
}
ignoredFields := sets.NewString("Transformer", "EncryptedDEK", "UID", "CacheKey")
ignoredFields := sets.NewString("Transformer", "EncryptedObject.EncryptedDEKSource", "UID", "CacheKey")
if diff := cmp.Diff(tt.wantState, *h.state.Load(),
gotState := *h.state.Load()
if diff := cmp.Diff(tt.wantState, gotState,
cmp.FilterPath(func(path cmp.Path) bool { return ignoredFields.Has(path.String()) }, cmp.Ignore()),
); len(diff) > 0 {
t.Errorf("state mismatch (-want +got):\n%s", diff)
}
if len(cmp.Diff(tt.wantState, gotState)) > 0 { // we only need to run this check when the state changes
validCiphertext := len(gotState.EncryptedObject.EncryptedDEKSource) > 0
if tt.useSeed {
validCiphertext = validCiphertext && gotState.EncryptedObject.EncryptedDEKSourceType == kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED
} else {
validCiphertext = validCiphertext && gotState.EncryptedObject.EncryptedDEKSourceType == kmstypes.EncryptedDEKSourceType_AES_GCM_KEY
}
if !validCiphertext {
t.Errorf("invalid ciphertext with useSeed=%v, encryptedDEKSourceLen=%d, encryptedDEKSourceType=%d", tt.useSeed,
len(gotState.EncryptedObject.EncryptedDEKSource), gotState.EncryptedObject.EncryptedDEKSourceType)
}
}
if tt.wantEncryptCalls != tt.service.encryptCalls {
t.Errorf("want %d encryptCalls, got %d", tt.wantEncryptCalls, tt.service.encryptCalls)
}
@@ -1900,15 +1937,15 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
func validState(t *testing.T, keyID string, exp time.Time) envelopekmsv2.State {
t.Helper()
transformer, resp, cacheKey, err := envelopekmsv2.GenerateTransformer(testContext(t), "", &testKMSv2EnvelopeService{keyID: keyID})
useSeed := utilfeature.DefaultFeatureGate.Enabled(features.KMSv2KDF) // match the current default behavior
transformer, encObject, cacheKey, err := envelopekmsv2.GenerateTransformer(testContext(t), "", &testKMSv2EnvelopeService{keyID: keyID}, useSeed)
if err != nil {
t.Fatal(err)
}
return envelopekmsv2.State{
Transformer: transformer,
EncryptedDEK: resp.Ciphertext,
KeyID: resp.KeyID,
Annotations: resp.Annotations,
EncryptedObject: *encObject,
ExpirationTimestamp: exp,
CacheKey: cacheKey,
}

View File

@@ -34,33 +34,11 @@ import (
"k8s.io/klog/v2"
)
type gcm struct {
aead cipher.AEAD
nonceFunc func([]byte) error
}
// commonSize is the length of various security sensitive byte slices such as encryption keys.
// Do not change this value. It would be a backward incompatible change.
const commonSize = 32
// NewGCMTransformer takes the given block cipher and performs encryption and decryption on the given data.
// It implements AEAD encryption of the provided values given a cipher.Block algorithm.
// The authenticated data provided as part of the value.Context method must match when the same
// value is set to and loaded from storage. In order to ensure that values cannot be copied by
// an attacker from a location under their control, use characteristics of the storage location
// (such as the etcd key) as part of the authenticated data.
//
// Because this mode requires a generated IV and IV reuse is a known weakness of AES-GCM, keys
// must be rotated before a birthday attack becomes feasible. NIST SP 800-38D
// (http://csrc.nist.gov/publications/nistpubs/800-38D/SP-800-38D.pdf) recommends using the same
// key with random 96-bit nonces (the default nonce length) no more than 2^32 times, and
// therefore transformers using this implementation *must* ensure they allow for frequent key
// rotation. Future work should include investigation of AES-GCM-SIV as an alternative to
// random nonces.
func NewGCMTransformer(block cipher.Block) (value.Transformer, error) {
aead, err := newGCM(block)
if err != nil {
return nil, err
}
return &gcm{aead: aead, nonceFunc: randomNonce}, nil
}
const keySizeCounterNonceGCM = commonSize
// NewGCMTransformerWithUniqueKeyUnsafe is the same as NewGCMTransformer but is unsafe for general
// use because it makes assumptions about the key underlying the block cipher. Specifically,
@@ -78,7 +56,7 @@ func NewGCMTransformer(block cipher.Block) (value.Transformer, error) {
// it can be passed to NewGCMTransformer(aes.NewCipher(key)) to construct a transformer capable
// of decrypting values encrypted by this transformer (that transformer must not be used for encryption).
func NewGCMTransformerWithUniqueKeyUnsafe() (value.Transformer, []byte, error) {
key, err := generateKey(32)
key, err := GenerateKey(keySizeCounterNonceGCM)
if err != nil {
return nil, nil, err
}
@@ -126,17 +104,6 @@ func newGCMTransformerWithUniqueKeyUnsafe(block cipher.Block, nonceGen *nonceGen
return &gcm{aead: aead, nonceFunc: nonceFunc}, nil
}
func newGCM(block cipher.Block) (cipher.AEAD, error) {
aead, err := cipher.NewGCM(block)
if err != nil {
return nil, err
}
if nonceSize := aead.NonceSize(); nonceSize != 12 { // all data in etcd will be broken if this ever changes
return nil, fmt.Errorf("crypto/cipher.NewGCM returned unexpected nonce size: %d", nonceSize)
}
return aead, nil
}
func randomNonce(b []byte) error {
_, err := rand.Read(b)
return err
@@ -164,8 +131,8 @@ func die(msg string) {
klog.FatalDepth(1, msg)
}
// generateKey generates a random key using system randomness.
func generateKey(length int) (key []byte, err error) {
// GenerateKey generates a random key using system randomness.
func GenerateKey(length int) (key []byte, err error) {
defer func(start time.Time) {
value.RecordDataKeyGeneration(start, err)
}(time.Now())
@@ -177,6 +144,45 @@ func generateKey(length int) (key []byte, err error) {
return key, nil
}
// NewGCMTransformer takes the given block cipher and performs encryption and decryption on the given data.
// It implements AEAD encryption of the provided values given a cipher.Block algorithm.
// The authenticated data provided as part of the value.Context method must match when the same
// value is set to and loaded from storage. In order to ensure that values cannot be copied by
// an attacker from a location under their control, use characteristics of the storage location
// (such as the etcd key) as part of the authenticated data.
//
// Because this mode requires a generated IV and IV reuse is a known weakness of AES-GCM, keys
// must be rotated before a birthday attack becomes feasible. NIST SP 800-38D
// (http://csrc.nist.gov/publications/nistpubs/800-38D/SP-800-38D.pdf) recommends using the same
// key with random 96-bit nonces (the default nonce length) no more than 2^32 times, and
// therefore transformers using this implementation *must* ensure they allow for frequent key
// rotation. Future work should include investigation of AES-GCM-SIV as an alternative to
// random nonces.
func NewGCMTransformer(block cipher.Block) (value.Transformer, error) {
aead, err := newGCM(block)
if err != nil {
return nil, err
}
return &gcm{aead: aead, nonceFunc: randomNonce}, nil
}
func newGCM(block cipher.Block) (cipher.AEAD, error) {
aead, err := cipher.NewGCM(block)
if err != nil {
return nil, err
}
if nonceSize := aead.NonceSize(); nonceSize != 12 { // all data in etcd will be broken if this ever changes
return nil, fmt.Errorf("crypto/cipher.NewGCM returned unexpected nonce size: %d", nonceSize)
}
return aead, nil
}
type gcm struct {
aead cipher.AEAD
nonceFunc func([]byte) error
}
func (t *gcm) TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, bool, error) {
nonceSize := t.aead.NonceSize()
if len(data) < nonceSize {

View File

@@ -0,0 +1,186 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aes
import (
"bytes"
"context"
"crypto/aes"
"crypto/sha256"
"errors"
"fmt"
"io"
"time"
"golang.org/x/crypto/hkdf"
"k8s.io/apiserver/pkg/storage/value"
"k8s.io/utils/clock"
)
const (
// cacheTTL is the TTL of KDF cache entries. We assume that the value.Context.AuthenticatedData
// for every call is the etcd storage path of the associated resource, and use that as the primary
// cache key (with a secondary check that confirms that the info matches). Thus if a client
// is constantly creating resources with new names (and thus new paths), they will keep adding new
// entries to the cache for up to this TTL before the GC logic starts deleting old entries. Each
// entry is ~300 bytes in size, so even a malicious client will be bounded in the overall memory
// it can consume.
cacheTTL = 10 * time.Minute
derivedKeySizeExtendedNonceGCM = commonSize
infoSizeExtendedNonceGCM
MinSeedSizeExtendedNonceGCM
)
// NewHKDFExtendedNonceGCMTransformer is the same as NewGCMTransformer but trades storage,
// memory and CPU to work around the limitations of AES-GCM's 12 byte nonce size. The input seed
// is assumed to be a cryptographically strong slice of MinSeedSizeExtendedNonceGCM+ random bytes.
// Unlike NewGCMTransformer, this function is immune to the birthday attack because a new key is generated
// per encryption via a key derivation function: KDF(seed, random_bytes) -> key. The derived key is
// only used once as an AES-GCM key with a random 12 byte nonce. This avoids any concerns around
// cryptographic wear out (by either number of encryptions or the amount of data being encrypted).
// Speaking on the cryptographic safety, the limit on the number of operations that can be preformed
// with a single seed with derived keys and randomly generated nonces is not practically reachable.
// Thus, the scheme does not impose any specific requirements on the seed rotation schedule.
// Reusing the same seed is safe to do over time and across process restarts. Whenever a new
// seed is needed, the caller should generate it via GenerateKey(MinSeedSizeExtendedNonceGCM).
// In regard to KMSv2, organization standards or compliance policies around rotation may require
// that the seed be rotated at some interval. This can be implemented externally by rotating
// the key encryption key via a key ID change.
func NewHKDFExtendedNonceGCMTransformer(seed []byte) (value.Transformer, error) {
if seedLen := len(seed); seedLen < MinSeedSizeExtendedNonceGCM {
return nil, fmt.Errorf("invalid seed length %d used for key generation", seedLen)
}
return &extendedNonceGCM{
seed: seed,
cache: newSimpleCache(clock.RealClock{}, cacheTTL),
}, nil
}
type extendedNonceGCM struct {
seed []byte
cache *simpleCache
}
func (e *extendedNonceGCM) TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, bool, error) {
if len(data) < infoSizeExtendedNonceGCM {
return nil, false, errors.New("the stored data was shorter than the required size")
}
info := data[:infoSizeExtendedNonceGCM]
transformer, err := e.derivedKeyTransformer(info, dataCtx, false)
if err != nil {
return nil, false, fmt.Errorf("failed to derive read key from KDF: %w", err)
}
return transformer.TransformFromStorage(ctx, data, dataCtx)
}
func (e *extendedNonceGCM) TransformToStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, error) {
info := make([]byte, infoSizeExtendedNonceGCM)
if err := randomNonce(info); err != nil {
return nil, fmt.Errorf("failed to generate info for KDF: %w", err)
}
transformer, err := e.derivedKeyTransformer(info, dataCtx, true)
if err != nil {
return nil, fmt.Errorf("failed to derive write key from KDF: %w", err)
}
return transformer.TransformToStorage(ctx, data, dataCtx)
}
func (e *extendedNonceGCM) derivedKeyTransformer(info []byte, dataCtx value.Context, write bool) (value.Transformer, error) {
if !write { // no need to check cache on write since we always generate a new transformer
if transformer := e.cache.get(info, dataCtx); transformer != nil {
return transformer, nil
}
// on read, this is a subslice of a much larger slice and we do not want to hold onto that larger slice
info = bytes.Clone(info)
}
key, err := e.sha256KDFExpandOnly(info)
if err != nil {
return nil, fmt.Errorf("failed to KDF expand seed with info: %w", err)
}
transformer, err := newGCMTransformerWithInfo(key, info)
if err != nil {
return nil, fmt.Errorf("failed to build transformer with KDF derived key: %w", err)
}
e.cache.set(dataCtx, transformer)
return transformer, nil
}
func (e *extendedNonceGCM) sha256KDFExpandOnly(info []byte) ([]byte, error) {
kdf := hkdf.Expand(sha256.New, e.seed, info)
derivedKey := make([]byte, derivedKeySizeExtendedNonceGCM)
if _, err := io.ReadFull(kdf, derivedKey); err != nil {
return nil, fmt.Errorf("failed to read a derived key from KDF: %w", err)
}
return derivedKey, nil
}
func newGCMTransformerWithInfo(key, info []byte) (*transformerWithInfo, error) {
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
transformer, err := NewGCMTransformer(block)
if err != nil {
return nil, err
}
return &transformerWithInfo{transformer: transformer, info: info}, nil
}
type transformerWithInfo struct {
transformer value.Transformer
// info are extra opaque bytes prepended to the writes from transformer and stripped from reads.
// currently info is used to generate a key via KDF(seed, info) -> key
// and transformer is the output of NewGCMTransformer(aes.NewCipher(key))
info []byte
}
func (t *transformerWithInfo) TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, bool, error) {
if !bytes.HasPrefix(data, t.info) {
return nil, false, errors.New("the stored data is missing the required info prefix")
}
return t.transformer.TransformFromStorage(ctx, data[len(t.info):], dataCtx)
}
func (t *transformerWithInfo) TransformToStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, error) {
out, err := t.transformer.TransformToStorage(ctx, data, dataCtx)
if err != nil {
return nil, err
}
outWithInfo := make([]byte, 0, len(out)+len(t.info))
outWithInfo = append(outWithInfo, t.info...)
outWithInfo = append(outWithInfo, out...)
return outWithInfo, nil
}

View File

@@ -152,7 +152,7 @@ func TestGCMUnsafeCompatibility(t *testing.T) {
t.Fatal(err)
}
transformerDecrypt := newGCMTransformer(t, block)
transformerDecrypt := newGCMTransformer(t, block, nil)
ctx := context.Background()
dataCtx := value.DefaultContext("authenticated_data")
@@ -184,7 +184,7 @@ func TestGCMLegacyDataCompatibility(t *testing.T) {
t.Fatal(err)
}
transformerDecrypt := newGCMTransformer(t, block)
transformerDecrypt := newGCMTransformer(t, block, nil)
// recorded output from NewGCMTransformer at commit 3b1fc60d8010dd8b53e97ba80e4710dbb430beee
const legacyCiphertext = "\x9f'\xc8\xfc\xea\x8aX\xc4g\xd8\xe47\xdb\xf2\xd8YU\xf9\xb4\xbd\x91/N\xf9g\u05c8\xa0\xcb\ay}\xac\n?\n\bE`\\\xa8Z\xc8V+J\xe1"
@@ -204,12 +204,36 @@ func TestGCMLegacyDataCompatibility(t *testing.T) {
}
}
func TestExtendedNonceGCMLegacyDataCompatibility(t *testing.T) {
// recorded output from NewKDFExtendedNonceGCMTransformerWithUniqueSeed from https://github.com/kubernetes/kubernetes/pull/118828
const (
legacyKey = "]@2:\x82\x0f\xf9Uag^;\x95\xe8\x18g\xc5\xfd\xd5a\xd3Z\x88\xa2Ћ\b\xaa\x9dO\xcf\\"
legacyCiphertext = "$Bu\x9e3\x94_\xba\xd7\t\xdbWz\x0f\x03\x7fا\t\xfcv\x97\x9b\x89B \x9d\xeb\xce˝W\xef\xe3\xd6\xffj\x1e\xf6\xee\x9aP\x03\xb9\x83;0C\xce\xc1\xe4{5\x17[\x15\x11\a\xa8\xd2Ak\x0e)k\xbff\xb5\xd1\x02\xfc\xefߚx\xf2\x93\xd2q"
)
transformerDecrypt := newHKDFExtendedNonceGCMTransformerTest(t, nil, []byte(legacyKey))
ctx := context.Background()
dataCtx := value.DefaultContext("bamboo")
plaintext := []byte("pandas are the best")
plaintextAgain, _, err := transformerDecrypt.TransformFromStorage(ctx, []byte(legacyCiphertext), dataCtx)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(plaintext, plaintextAgain) {
t.Errorf("expected original plaintext %q, got %q", string(plaintext), string(plaintextAgain))
}
}
func TestGCMUnsafeNonceGen(t *testing.T) {
block, err := aes.NewCipher([]byte("abcdefghijklmnop"))
if err != nil {
t.Fatal(err)
}
transformer := newGCMTransformerWithUniqueKeyUnsafeTest(t, block)
transformer := newGCMTransformerWithUniqueKeyUnsafeTest(t, block, nil)
ctx := context.Background()
dataCtx := value.DefaultContext("authenticated_data")
@@ -270,7 +294,7 @@ func TestGCMUnsafeNonceGen(t *testing.T) {
func TestGCMNonce(t *testing.T) {
t.Run("gcm", func(t *testing.T) {
testGCMNonce(t, newGCMTransformer, func(_ int, nonce []byte) {
testGCMNonce(t, newGCMTransformer, 0, func(_ int, nonce []byte) {
if bytes.Equal(nonce, make([]byte, len(nonce))) {
t.Error("got all zeros for nonce")
}
@@ -278,21 +302,30 @@ func TestGCMNonce(t *testing.T) {
})
t.Run("gcm unsafe", func(t *testing.T) {
testGCMNonce(t, newGCMTransformerWithUniqueKeyUnsafeTest, func(i int, nonce []byte) {
testGCMNonce(t, newGCMTransformerWithUniqueKeyUnsafeTest, 0, func(i int, nonce []byte) {
counter := binary.LittleEndian.Uint64(nonce)
if uint64(i+1) != counter { // add one because the counter starts at 1, not 0
t.Errorf("counter nonce is invalid: want %d, got %d", i+1, counter)
}
})
})
t.Run("gcm extended nonce", func(t *testing.T) {
testGCMNonce(t, newHKDFExtendedNonceGCMTransformerTest, infoSizeExtendedNonceGCM, func(_ int, nonce []byte) {
if bytes.Equal(nonce, make([]byte, len(nonce))) {
t.Error("got all zeros for nonce")
}
})
})
}
func testGCMNonce(t *testing.T, f func(t testingT, block cipher.Block) value.Transformer, check func(int, []byte)) {
block, err := aes.NewCipher([]byte("abcdefghijklmnop"))
func testGCMNonce(t *testing.T, f transformerFunc, infoLen int, check func(int, []byte)) {
key := []byte("abcdefghijklmnopabcdefghijklmnop")
block, err := aes.NewCipher(key)
if err != nil {
t.Fatal(err)
}
transformer := f(t, block)
transformer := f(t, block, key)
ctx := context.Background()
dataCtx := value.DefaultContext("authenticated_data")
@@ -307,13 +340,20 @@ func testGCMNonce(t *testing.T, f func(t testingT, block cipher.Block) value.Tra
t.Fatal(err)
}
nonce := out[:12]
info := out[:infoLen]
nonce := out[infoLen : 12+infoLen]
randomN := nonce[:4]
if bytes.Equal(randomN, make([]byte, len(randomN))) {
t.Error("got all zeros for first four bytes")
}
if infoLen != 0 {
if bytes.Equal(info, make([]byte, infoLen)) {
t.Error("got all zeros for info")
}
}
check(i, nonce[4:])
}
}
@@ -326,15 +366,22 @@ func TestGCMKeyRotation(t *testing.T) {
t.Run("gcm unsafe", func(t *testing.T) {
testGCMKeyRotation(t, newGCMTransformerWithUniqueKeyUnsafeTest)
})
t.Run("gcm extended", func(t *testing.T) {
testGCMKeyRotation(t, newHKDFExtendedNonceGCMTransformerTest)
})
}
func testGCMKeyRotation(t *testing.T, f func(t testingT, block cipher.Block) value.Transformer) {
func testGCMKeyRotation(t *testing.T, f transformerFunc) {
key1 := []byte("abcdefghijklmnopabcdefghijklmnop")
key2 := []byte("0123456789abcdef0123456789abcdef")
testErr := fmt.Errorf("test error")
block1, err := aes.NewCipher([]byte("abcdefghijklmnop"))
block1, err := aes.NewCipher(key1)
if err != nil {
t.Fatal(err)
}
block2, err := aes.NewCipher([]byte("0123456789abcdef"))
block2, err := aes.NewCipher(key2)
if err != nil {
t.Fatal(err)
}
@@ -343,8 +390,8 @@ func testGCMKeyRotation(t *testing.T, f func(t testingT, block cipher.Block) val
dataCtx := value.DefaultContext("authenticated_data")
p := value.NewPrefixTransformers(testErr,
value.PrefixTransformer{Prefix: []byte("first:"), Transformer: f(t, block1)},
value.PrefixTransformer{Prefix: []byte("second:"), Transformer: f(t, block2)},
value.PrefixTransformer{Prefix: []byte("first:"), Transformer: f(t, block1, key1)},
value.PrefixTransformer{Prefix: []byte("second:"), Transformer: f(t, block2, key2)},
)
out, err := p.TransformToStorage(ctx, []byte("firstvalue"), dataCtx)
if err != nil {
@@ -369,8 +416,8 @@ func testGCMKeyRotation(t *testing.T, f func(t testingT, block cipher.Block) val
// reverse the order, use the second key
p = value.NewPrefixTransformers(testErr,
value.PrefixTransformer{Prefix: []byte("second:"), Transformer: f(t, block2)},
value.PrefixTransformer{Prefix: []byte("first:"), Transformer: f(t, block1)},
value.PrefixTransformer{Prefix: []byte("second:"), Transformer: f(t, block2, key2)},
value.PrefixTransformer{Prefix: []byte("first:"), Transformer: f(t, block1, key1)},
)
from, stale, err = p.TransformFromStorage(ctx, out, dataCtx)
if err != nil {
@@ -434,6 +481,12 @@ func TestCBCKeyRotation(t *testing.T) {
}
}
var gcmBenchmarks = []namedTransformerFunc{
{name: "gcm-random-nonce", f: newGCMTransformer},
{name: "gcm-counter-nonce", f: newGCMTransformerWithUniqueKeyUnsafeTest},
{name: "gcm-extended-nonce", f: newHKDFExtendedNonceGCMTransformerTest},
}
func BenchmarkGCMRead(b *testing.B) {
tests := []struct {
keyLength int
@@ -448,7 +501,16 @@ func BenchmarkGCMRead(b *testing.B) {
for _, t := range tests {
name := fmt.Sprintf("%vKeyLength/%vValueLength/%vExpectStale", t.keyLength, t.valueLength, t.expectStale)
b.Run(name, func(b *testing.B) {
benchmarkGCMRead(b, t.keyLength, t.valueLength, t.expectStale)
for _, n := range gcmBenchmarks {
n := n
if t.keyLength == 16 && n.name == "gcm-extended-nonce" {
continue // gcm-extended-nonce requires 32 byte keys
}
b.Run(n.name, func(b *testing.B) {
b.ReportAllocs()
benchmarkGCMRead(b, n.f, t.keyLength, t.valueLength, t.expectStale)
})
}
})
}
}
@@ -465,23 +527,35 @@ func BenchmarkGCMWrite(b *testing.B) {
for _, t := range tests {
name := fmt.Sprintf("%vKeyLength/%vValueLength", t.keyLength, t.valueLength)
b.Run(name, func(b *testing.B) {
benchmarkGCMWrite(b, t.keyLength, t.valueLength)
for _, n := range gcmBenchmarks {
n := n
if t.keyLength == 16 && n.name == "gcm-extended-nonce" {
continue // gcm-extended-nonce requires 32 byte keys
}
b.Run(n.name, func(b *testing.B) {
b.ReportAllocs()
benchmarkGCMWrite(b, n.f, t.keyLength, t.valueLength)
})
}
})
}
}
func benchmarkGCMRead(b *testing.B, keyLength int, valueLength int, expectStale bool) {
block1, err := aes.NewCipher(bytes.Repeat([]byte("a"), keyLength))
func benchmarkGCMRead(b *testing.B, f transformerFunc, keyLength int, valueLength int, expectStale bool) {
key1 := bytes.Repeat([]byte("a"), keyLength)
key2 := bytes.Repeat([]byte("b"), keyLength)
block1, err := aes.NewCipher(key1)
if err != nil {
b.Fatal(err)
}
block2, err := aes.NewCipher(bytes.Repeat([]byte("b"), keyLength))
block2, err := aes.NewCipher(key2)
if err != nil {
b.Fatal(err)
}
p := value.NewPrefixTransformers(nil,
value.PrefixTransformer{Prefix: []byte("first:"), Transformer: newGCMTransformer(b, block1)},
value.PrefixTransformer{Prefix: []byte("second:"), Transformer: newGCMTransformer(b, block2)},
value.PrefixTransformer{Prefix: []byte("first:"), Transformer: f(b, block1, key1)},
value.PrefixTransformer{Prefix: []byte("second:"), Transformer: f(b, block2, key2)},
)
ctx := context.Background()
@@ -495,8 +569,8 @@ func benchmarkGCMRead(b *testing.B, keyLength int, valueLength int, expectStale
// reverse the key order if expecting stale
if expectStale {
p = value.NewPrefixTransformers(nil,
value.PrefixTransformer{Prefix: []byte("second:"), Transformer: newGCMTransformer(b, block2)},
value.PrefixTransformer{Prefix: []byte("first:"), Transformer: newGCMTransformer(b, block1)},
value.PrefixTransformer{Prefix: []byte("second:"), Transformer: f(b, block2, key2)},
value.PrefixTransformer{Prefix: []byte("first:"), Transformer: f(b, block1, key1)},
)
}
@@ -513,18 +587,21 @@ func benchmarkGCMRead(b *testing.B, keyLength int, valueLength int, expectStale
b.StopTimer()
}
func benchmarkGCMWrite(b *testing.B, keyLength int, valueLength int) {
block1, err := aes.NewCipher(bytes.Repeat([]byte("a"), keyLength))
func benchmarkGCMWrite(b *testing.B, f transformerFunc, keyLength int, valueLength int) {
key1 := bytes.Repeat([]byte("a"), keyLength)
key2 := bytes.Repeat([]byte("b"), keyLength)
block1, err := aes.NewCipher(key1)
if err != nil {
b.Fatal(err)
}
block2, err := aes.NewCipher(bytes.Repeat([]byte("b"), keyLength))
block2, err := aes.NewCipher(key2)
if err != nil {
b.Fatal(err)
}
p := value.NewPrefixTransformers(nil,
value.PrefixTransformer{Prefix: []byte("first:"), Transformer: newGCMTransformer(b, block1)},
value.PrefixTransformer{Prefix: []byte("second:"), Transformer: newGCMTransformer(b, block2)},
value.PrefixTransformer{Prefix: []byte("first:"), Transformer: f(b, block1, key1)},
value.PrefixTransformer{Prefix: []byte("second:"), Transformer: f(b, block2, key2)},
)
ctx := context.Background()
@@ -657,31 +734,29 @@ func TestRoundTrip(t *testing.T) {
if err != nil {
t.Fatal(err)
}
aes32block, err := aes.NewCipher(bytes.Repeat([]byte("c"), 32))
key32 := bytes.Repeat([]byte("c"), 32)
aes32block, err := aes.NewCipher(key32)
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
tests := []struct {
name string
dataCtx value.Context
t value.Transformer
name string
t value.Transformer
}{
{name: "GCM 16 byte key", t: newGCMTransformer(t, aes16block)},
{name: "GCM 24 byte key", t: newGCMTransformer(t, aes24block)},
{name: "GCM 32 byte key", t: newGCMTransformer(t, aes32block)},
{name: "GCM 16 byte unsafe key", t: newGCMTransformerWithUniqueKeyUnsafeTest(t, aes16block)},
{name: "GCM 24 byte unsafe key", t: newGCMTransformerWithUniqueKeyUnsafeTest(t, aes24block)},
{name: "GCM 32 byte unsafe key", t: newGCMTransformerWithUniqueKeyUnsafeTest(t, aes32block)},
{name: "GCM 16 byte key", t: newGCMTransformer(t, aes16block, nil)},
{name: "GCM 24 byte key", t: newGCMTransformer(t, aes24block, nil)},
{name: "GCM 32 byte key", t: newGCMTransformer(t, aes32block, nil)},
{name: "GCM 16 byte unsafe key", t: newGCMTransformerWithUniqueKeyUnsafeTest(t, aes16block, nil)},
{name: "GCM 24 byte unsafe key", t: newGCMTransformerWithUniqueKeyUnsafeTest(t, aes24block, nil)},
{name: "GCM 32 byte unsafe key", t: newGCMTransformerWithUniqueKeyUnsafeTest(t, aes32block, nil)},
{name: "GCM 32 byte seed", t: newHKDFExtendedNonceGCMTransformerTest(t, nil, key32)},
{name: "CBC 32 byte key", t: NewCBCTransformer(aes32block)},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
dataCtx := tt.dataCtx
if dataCtx == nil {
dataCtx = value.DefaultContext("")
}
dataCtx := value.DefaultContext("/foo/bar")
for _, l := range lengths {
data := make([]byte, l)
if _, err := io.ReadFull(rand.Reader, data); err != nil {
@@ -718,12 +793,14 @@ func TestRoundTrip(t *testing.T) {
}
}
type testingT interface {
Helper()
Fatal(...any)
type namedTransformerFunc struct {
name string
f transformerFunc
}
func newGCMTransformer(t testingT, block cipher.Block) value.Transformer {
type transformerFunc func(t testing.TB, block cipher.Block, key []byte) value.Transformer
func newGCMTransformer(t testing.TB, block cipher.Block, _ []byte) value.Transformer {
t.Helper()
transformer, err := NewGCMTransformer(block)
@@ -734,7 +811,7 @@ func newGCMTransformer(t testingT, block cipher.Block) value.Transformer {
return transformer
}
func newGCMTransformerWithUniqueKeyUnsafeTest(t testingT, block cipher.Block) value.Transformer {
func newGCMTransformerWithUniqueKeyUnsafeTest(t testing.TB, block cipher.Block, _ []byte) value.Transformer {
t.Helper()
nonceGen := &nonceGenerator{fatal: die}
@@ -745,3 +822,14 @@ func newGCMTransformerWithUniqueKeyUnsafeTest(t testingT, block cipher.Block) va
return transformer
}
func newHKDFExtendedNonceGCMTransformerTest(t testing.TB, _ cipher.Block, key []byte) value.Transformer {
t.Helper()
transformer, err := NewHKDFExtendedNonceGCMTransformer(key)
if err != nil {
t.Fatal(err)
}
return transformer
}

View File

@@ -0,0 +1,91 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aes
import (
"bytes"
"time"
"unsafe"
utilcache "k8s.io/apimachinery/pkg/util/cache"
"k8s.io/apiserver/pkg/storage/value"
"k8s.io/utils/clock"
)
type simpleCache struct {
cache *utilcache.Expiring
ttl time.Duration
}
func newSimpleCache(clock clock.Clock, ttl time.Duration) *simpleCache {
cache := utilcache.NewExpiringWithClock(clock)
// "Stale" entries are always valid for us because the TTL is just used to prevent
// unbounded growth on the cache - for a given info the transformer is always the same.
// The key always corresponds to the exact same value, with the caveat that
// since we use the value.Context.AuthenticatedData to overwrite old keys,
// we always have to check that the info matches (to validate the transformer is correct).
cache.AllowExpiredGet = true
return &simpleCache{
cache: cache,
ttl: ttl,
}
}
// given a key, return the transformer, or nil if it does not exist in the cache
func (c *simpleCache) get(info []byte, dataCtx value.Context) *transformerWithInfo {
val, ok := c.cache.Get(keyFunc(dataCtx))
if !ok {
return nil
}
transformer := val.(*transformerWithInfo)
if !bytes.Equal(transformer.info, info) {
return nil
}
return transformer
}
// set caches the record for the key
func (c *simpleCache) set(dataCtx value.Context, transformer *transformerWithInfo) {
if dataCtx == nil || len(dataCtx.AuthenticatedData()) == 0 {
panic("authenticated data must not be empty")
}
if transformer == nil {
panic("transformer must not be nil")
}
if len(transformer.info) == 0 {
panic("info must not be empty")
}
c.cache.Set(keyFunc(dataCtx), transformer, c.ttl)
}
func keyFunc(dataCtx value.Context) string {
return toString(dataCtx.AuthenticatedData())
}
// toString performs unholy acts to avoid allocations
func toString(b []byte) string {
// unsafe.SliceData relies on cap whereas we want to rely on len
if len(b) == 0 {
return ""
}
// Copied from go 1.20.1 strings.Builder.String
// https://github.com/golang/go/blob/202a1a57064127c3f19d96df57b9f9586145e21c/src/strings/builder.go#L48
return unsafe.String(unsafe.SliceData(b), len(b))
}

View File

@@ -0,0 +1,137 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aes
import (
"testing"
"time"
clocktesting "k8s.io/utils/clock/testing"
)
type dataString string
func (d dataString) AuthenticatedData() []byte { return []byte(d) }
func Test_simpleCache(t *testing.T) {
info1 := []byte{1}
info2 := []byte{2}
key1 := dataString("1")
key2 := dataString("2")
twi1 := &transformerWithInfo{info: info1}
twi2 := &transformerWithInfo{info: info2}
tests := []struct {
name string
test func(*testing.T, *simpleCache, *clocktesting.FakeClock)
}{
{
name: "get from empty",
test: func(t *testing.T, cache *simpleCache, clock *clocktesting.FakeClock) {
got := cache.get(info1, key1)
twiPtrEquals(t, nil, got)
cacheLenEquals(t, cache, 0)
},
},
{
name: "get after set",
test: func(t *testing.T, cache *simpleCache, clock *clocktesting.FakeClock) {
cache.set(key1, twi1)
got := cache.get(info1, key1)
twiPtrEquals(t, twi1, got)
cacheLenEquals(t, cache, 1)
},
},
{
name: "get after set but with different info",
test: func(t *testing.T, cache *simpleCache, clock *clocktesting.FakeClock) {
cache.set(key1, twi1)
got := cache.get(info2, key1)
twiPtrEquals(t, nil, got)
cacheLenEquals(t, cache, 1)
},
},
{
name: "expired get after set",
test: func(t *testing.T, cache *simpleCache, clock *clocktesting.FakeClock) {
cache.set(key1, twi1)
clock.Step(time.Hour)
got := cache.get(info1, key1)
twiPtrEquals(t, twi1, got)
cacheLenEquals(t, cache, 1)
},
},
{
name: "expired get after GC",
test: func(t *testing.T, cache *simpleCache, clock *clocktesting.FakeClock) {
cache.set(key1, twi1)
clock.Step(time.Hour)
cacheLenEquals(t, cache, 1)
cache.set(key2, twi2) // unrelated set to make GC run
got := cache.get(info1, key1)
twiPtrEquals(t, nil, got)
cacheLenEquals(t, cache, 1)
},
},
{
name: "multiple sets for same key",
test: func(t *testing.T, cache *simpleCache, clock *clocktesting.FakeClock) {
cache.set(key1, twi1)
cacheLenEquals(t, cache, 1)
cache.set(key1, twi2)
cacheLenEquals(t, cache, 1)
got11 := cache.get(info1, key1)
twiPtrEquals(t, nil, got11)
got21 := cache.get(info2, key1)
twiPtrEquals(t, twi2, got21)
got12 := cache.get(info1, key2)
twiPtrEquals(t, nil, got12)
got22 := cache.get(info2, key2)
twiPtrEquals(t, nil, got22)
},
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
clock := clocktesting.NewFakeClock(time.Now())
cache := newSimpleCache(clock, 10*time.Second)
tt.test(t, cache, clock)
})
}
}
func twiPtrEquals(t *testing.T, want, got *transformerWithInfo) {
t.Helper()
if want != got {
t.Errorf("transformerWithInfo structs are not pointer equivalent")
}
}
func cacheLenEquals(t *testing.T, cache *simpleCache, want int) {
t.Helper()
if got := cache.cache.Len(); want != got {
t.Errorf("unexpected cache len: want %d, got %d", want, got)
}
}

View File

@@ -67,7 +67,7 @@ func TestKeyFunc(t *testing.T) {
cache := newSimpleCache(fakeClock, time.Second)
t.Run("AllocsPerRun test", func(t *testing.T) {
key, err := generateKey(encryptedDEKMaxSize) // simulate worst case EDEK
key, err := generateKey(encryptedDEKSourceMaxSize) // simulate worst case EDEK
if err != nil {
t.Fatal(err)
}

View File

@@ -20,6 +20,7 @@ package kmsv2
import (
"context"
"crypto/aes"
"crypto/cipher"
"crypto/sha256"
"fmt"
"sort"
@@ -43,6 +44,8 @@ import (
"k8s.io/utils/clock"
)
// TODO integration test with old AES GCM data recorded and new KDF data recorded
func init() {
value.RegisterMetrics()
metrics.RegisterMetrics()
@@ -55,22 +58,22 @@ const (
annotationsMaxSize = 32 * 1024 // 32 kB
// KeyIDMaxSize is the maximum size of the keyID.
KeyIDMaxSize = 1 * 1024 // 1 kB
// encryptedDEKMaxSize is the maximum size of the encrypted DEK.
encryptedDEKMaxSize = 1 * 1024 // 1 kB
// encryptedDEKSourceMaxSize is the maximum size of the encrypted DEK source.
encryptedDEKSourceMaxSize = 1 * 1024 // 1 kB
// cacheTTL is the default time-to-live for the cache entry.
// this allows the cache to grow to an infinite size for up to a day.
// this is meant as a temporary solution until the cache is re-written to not have a TTL.
// there is unlikely to be any meaningful memory impact on the server
// because the cache will likely never have more than a few thousand entries
// and each entry is roughly ~200 bytes in size. with DEK reuse
// and no storage migration, the number of entries in this cache
// because the cache will likely never have more than a few thousand entries.
// each entry can be large due to an internal cache that maps the DEK seed to individual
// DEK entries, but that cache has an aggressive TTL to keep the size under control.
// with DEK/seed reuse and no storage migration, the number of entries in this cache
// would be approximated by unique key IDs used by the KMS plugin
// combined with the number of server restarts. If storage migration
// is performed after key ID changes, and the number of restarts
// is limited, this cache size may be as small as the number of API
// servers in use (once old entries expire out from the TTL).
cacheTTL = 24 * time.Hour
// error code
// key ID related error codes for metrics
errKeyIDOKCode ErrCodeKeyID = "ok"
errKeyIDEmptyCode ErrCodeKeyID = "empty"
errKeyIDTooLongCode ErrCodeKeyID = "too_long"
@@ -83,23 +86,22 @@ type StateFunc func() (State, error)
type ErrCodeKeyID string
type State struct {
Transformer value.Transformer
EncryptedDEK []byte
KeyID string
Annotations map[string][]byte
Transformer value.Transformer
EncryptedObject kmstypes.EncryptedObject
UID string
ExpirationTimestamp time.Time
// CacheKey is the key used to cache the DEK in transformer.cache.
// CacheKey is the key used to cache the DEK/seed in envelopeTransformer.cache.
CacheKey []byte
}
func (s *State) ValidateEncryptCapability() error {
if now := NowFunc(); now.After(s.ExpirationTimestamp) {
return fmt.Errorf("EDEK with keyID hash %q expired at %s (current time is %s)",
GetHashIfNotEmpty(s.KeyID), s.ExpirationTimestamp.Format(time.RFC3339), now.Format(time.RFC3339))
return fmt.Errorf("encryptedDEKSource with keyID hash %q expired at %s (current time is %s)",
GetHashIfNotEmpty(s.EncryptedObject.KeyID), s.ExpirationTimestamp.Format(time.RFC3339), now.Format(time.RFC3339))
}
return nil
}
@@ -137,6 +139,8 @@ func (t *envelopeTransformer) TransformFromStorage(ctx context.Context, data []b
return nil, false, err
}
useSeed := encryptedObject.EncryptedDEKSourceType == kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED
// TODO: consider marking state.EncryptedDEK != encryptedObject.EncryptedDEK as a stale read to support DEK defragmentation
// at a minimum we should have a metric that helps the user understand if DEK fragmentation is high
state, err := t.stateFunc() // no need to call state.ValidateEncryptCapability on reads
@@ -144,7 +148,7 @@ func (t *envelopeTransformer) TransformFromStorage(ctx context.Context, data []b
return nil, false, err
}
encryptedObjectCacheKey, err := generateCacheKey(encryptedObject.EncryptedDEK, encryptedObject.KeyID, encryptedObject.Annotations)
encryptedObjectCacheKey, err := generateCacheKey(encryptedObject.EncryptedDEKSourceType, encryptedObject.EncryptedDEKSource, encryptedObject.KeyID, encryptedObject.Annotations)
if err != nil {
return nil, false, err
}
@@ -163,7 +167,7 @@ func (t *envelopeTransformer) TransformFromStorage(ctx context.Context, data []b
"verb", requestInfo.Verb, "namespace", requestInfo.Namespace, "name", requestInfo.Name)
key, err := t.envelopeService.Decrypt(ctx, uid, &kmsservice.DecryptRequest{
Ciphertext: encryptedObject.EncryptedDEK,
Ciphertext: encryptedObject.EncryptedDEKSource,
KeyID: encryptedObject.KeyID,
Annotations: encryptedObject.Annotations,
})
@@ -171,7 +175,7 @@ func (t *envelopeTransformer) TransformFromStorage(ctx context.Context, data []b
return nil, false, fmt.Errorf("failed to decrypt DEK, error: %w", err)
}
transformer, err = t.addTransformerForDecryption(encryptedObjectCacheKey, key)
transformer, err = t.addTransformerForDecryption(encryptedObjectCacheKey, key, useSeed)
if err != nil {
return nil, false, err
}
@@ -184,8 +188,11 @@ func (t *envelopeTransformer) TransformFromStorage(ctx context.Context, data []b
}
// data is considered stale if the key ID does not match our current write transformer
return out, stale || encryptedObject.KeyID != state.KeyID, nil
return out,
stale ||
encryptedObject.KeyID != state.EncryptedObject.KeyID ||
encryptedObject.EncryptedDEKSourceType != state.EncryptedObject.EncryptedDEKSourceType,
nil
}
// TransformToStorage encrypts data to be written to disk using envelope encryption.
@@ -201,7 +208,7 @@ func (t *envelopeTransformer) TransformToStorage(ctx context.Context, data []byt
// this prevents a cache miss every time the DEK rotates
// this has the side benefit of causing the cache to perform a GC
// TODO see if we can do this inside the stateFunc control loop
// TODO(aramase): Add metrics for cache fill percentage with custom cache implementation.
// TODO(aramase): Add metrics for cache size.
t.cache.set(state.CacheKey, state.Transformer)
requestInfo := getRequestInfoFromContext(ctx)
@@ -214,39 +221,43 @@ func (t *envelopeTransformer) TransformToStorage(ctx context.Context, data []byt
return nil, err
}
metrics.RecordKeyID(metrics.ToStorageLabel, t.providerName, state.KeyID)
metrics.RecordKeyID(metrics.ToStorageLabel, t.providerName, state.EncryptedObject.KeyID)
encObject := &kmstypes.EncryptedObject{
KeyID: state.KeyID,
EncryptedDEK: state.EncryptedDEK,
EncryptedData: result,
Annotations: state.Annotations,
}
encObjectCopy := state.EncryptedObject
encObjectCopy.EncryptedData = result
// Serialize the EncryptedObject to a byte array.
return t.doEncode(encObject)
return t.doEncode(&encObjectCopy)
}
// addTransformerForDecryption inserts a new transformer to the Envelope cache of DEKs for future reads.
func (t *envelopeTransformer) addTransformerForDecryption(cacheKey []byte, key []byte) (value.Read, error) {
block, err := aes.NewCipher(key)
func (t *envelopeTransformer) addTransformerForDecryption(cacheKey []byte, key []byte, useSeed bool) (value.Read, error) {
var transformer value.Read
var err error
if useSeed {
// the input key is considered safe to use here because it is coming from the KMS plugin / etcd
transformer, err = aestransformer.NewHKDFExtendedNonceGCMTransformer(key)
} else {
var block cipher.Block
block, err = aes.NewCipher(key)
if err != nil {
return nil, err
}
// this is compatible with NewGCMTransformerWithUniqueKeyUnsafe for decryption
// it would use random nonces for encryption but we never do that
transformer, err = aestransformer.NewGCMTransformer(block)
}
if err != nil {
return nil, err
}
// this is compatible with NewGCMTransformerWithUniqueKeyUnsafe for decryption
// it would use random nonces for encryption but we never do that
transformer, err := aestransformer.NewGCMTransformer(block)
if err != nil {
return nil, err
}
// TODO(aramase): Add metrics for cache fill percentage with custom cache implementation.
// TODO(aramase): Add metrics for cache size.
t.cache.set(cacheKey, transformer)
return transformer, nil
}
// doEncode encodes the EncryptedObject to a byte array.
func (t *envelopeTransformer) doEncode(request *kmstypes.EncryptedObject) ([]byte, error) {
if err := validateEncryptedObject(request); err != nil {
if err := ValidateEncryptedObject(request); err != nil {
return nil, err
}
return proto.Marshal(request)
@@ -258,18 +269,31 @@ func (t *envelopeTransformer) doDecode(originalData []byte) (*kmstypes.Encrypted
if err := proto.Unmarshal(originalData, o); err != nil {
return nil, err
}
// validate the EncryptedObject
if err := validateEncryptedObject(o); err != nil {
if err := ValidateEncryptedObject(o); err != nil {
return nil, err
}
return o, nil
}
// GenerateTransformer generates a new transformer and encrypts the DEK using the envelope service.
// It returns the transformer, the encrypted DEK, cache key and error.
func GenerateTransformer(ctx context.Context, uid string, envelopeService kmsservice.Service) (value.Transformer, *kmsservice.EncryptResponse, []byte, error) {
transformer, newKey, err := aestransformer.NewGCMTransformerWithUniqueKeyUnsafe()
// GenerateTransformer generates a new transformer and encrypts the DEK/seed using the envelope service.
// It returns the transformer, the encrypted DEK/seed, cache key and error.
func GenerateTransformer(ctx context.Context, uid string, envelopeService kmsservice.Service, useSeed bool) (value.Transformer, *kmstypes.EncryptedObject, []byte, error) {
newTransformerFunc := func() (value.Transformer, []byte, error) {
seed, err := aestransformer.GenerateKey(aestransformer.MinSeedSizeExtendedNonceGCM)
if err != nil {
return nil, nil, err
}
transformer, err := aestransformer.NewHKDFExtendedNonceGCMTransformer(seed)
if err != nil {
return nil, nil, err
}
return transformer, seed, nil
}
if !useSeed {
newTransformerFunc = aestransformer.NewGCMTransformerWithUniqueKeyUnsafe
}
transformer, newKey, err := newTransformerFunc()
if err != nil {
return nil, nil, nil, err
}
@@ -281,32 +305,48 @@ func GenerateTransformer(ctx context.Context, uid string, envelopeService kmsser
return nil, nil, nil, fmt.Errorf("failed to encrypt DEK, error: %w", err)
}
if err := validateEncryptedObject(&kmstypes.EncryptedObject{
KeyID: resp.KeyID,
EncryptedDEK: resp.Ciphertext,
EncryptedData: []byte{0}, // any non-empty value to pass validation
Annotations: resp.Annotations,
}); err != nil {
o := &kmstypes.EncryptedObject{
KeyID: resp.KeyID,
EncryptedDEKSource: resp.Ciphertext,
EncryptedData: []byte{0}, // any non-empty value to pass validation
Annotations: resp.Annotations,
}
if useSeed {
o.EncryptedDEKSourceType = kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED
} else {
o.EncryptedDEKSourceType = kmstypes.EncryptedDEKSourceType_AES_GCM_KEY
}
if err := ValidateEncryptedObject(o); err != nil {
return nil, nil, nil, err
}
cacheKey, err := generateCacheKey(resp.Ciphertext, resp.KeyID, resp.Annotations)
cacheKey, err := generateCacheKey(o.EncryptedDEKSourceType, resp.Ciphertext, resp.KeyID, resp.Annotations)
if err != nil {
return nil, nil, nil, err
}
return transformer, resp, cacheKey, nil
o.EncryptedData = nil // make sure that later code that uses this encrypted object sets this field
return transformer, o, cacheKey, nil
}
func validateEncryptedObject(o *kmstypes.EncryptedObject) error {
func ValidateEncryptedObject(o *kmstypes.EncryptedObject) error {
if o == nil {
return fmt.Errorf("encrypted object is nil")
}
switch t := o.EncryptedDEKSourceType; t {
case kmstypes.EncryptedDEKSourceType_AES_GCM_KEY:
case kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED:
default:
return fmt.Errorf("unknown encryptedDEKSourceType: %d", t)
}
if len(o.EncryptedData) == 0 {
return fmt.Errorf("encrypted data is empty")
}
if err := validateEncryptedDEK(o.EncryptedDEK); err != nil {
return fmt.Errorf("failed to validate encrypted DEK: %w", err)
if err := validateEncryptedDEKSource(o.EncryptedDEKSource); err != nil {
return fmt.Errorf("failed to validate encrypted DEK source: %w", err)
}
if _, err := ValidateKeyID(o.KeyID); err != nil {
return fmt.Errorf("failed to validate key id: %w", err)
@@ -317,15 +357,15 @@ func validateEncryptedObject(o *kmstypes.EncryptedObject) error {
return nil
}
// validateEncryptedDEK tests the following:
// 1. The encrypted DEK is not empty.
// 2. The size of encrypted DEK is less than 1 kB.
func validateEncryptedDEK(encryptedDEK []byte) error {
if len(encryptedDEK) == 0 {
return fmt.Errorf("encrypted DEK is empty")
// validateEncryptedDEKSource tests the following:
// 1. The encrypted DEK source is not empty.
// 2. The size of encrypted DEK source is less than 1 kB.
func validateEncryptedDEKSource(encryptedDEKSource []byte) error {
if len(encryptedDEKSource) == 0 {
return fmt.Errorf("encrypted DEK source is empty")
}
if len(encryptedDEK) > encryptedDEKMaxSize {
return fmt.Errorf("encrypted DEK is %d bytes, which exceeds the max size of %d", len(encryptedDEK), encryptedDEKMaxSize)
if len(encryptedDEKSource) > encryptedDEKSourceMaxSize {
return fmt.Errorf("encrypted DEK source is %d bytes, which exceeds the max size of %d", len(encryptedDEKSource), encryptedDEKSourceMaxSize)
}
return nil
}
@@ -370,17 +410,19 @@ func getRequestInfoFromContext(ctx context.Context) *genericapirequest.RequestIn
// generateCacheKey returns a key for the cache.
// The key is a concatenation of:
// 1. encryptedDEK
// 0. encryptedDEKSourceType
// 1. encryptedDEKSource
// 2. keyID
// 3. length of annotations
// 4. annotations (sorted by key) - each annotation is a concatenation of:
// a. annotation key
// b. annotation value
func generateCacheKey(encryptedDEK []byte, keyID string, annotations map[string][]byte) ([]byte, error) {
func generateCacheKey(encryptedDEKSourceType kmstypes.EncryptedDEKSourceType, encryptedDEKSource []byte, keyID string, annotations map[string][]byte) ([]byte, error) {
// TODO(aramase): use sync pool buffer to avoid allocations
b := cryptobyte.NewBuilder(nil)
b.AddUint32(uint32(encryptedDEKSourceType))
b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
b.AddBytes(encryptedDEK)
b.AddBytes(encryptedDEKSource)
})
b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
b.AddBytes(toBytes(keyID))

View File

@@ -27,11 +27,13 @@ import (
"regexp"
"strconv"
"strings"
"sync"
"testing"
"time"
"github.com/gogo/protobuf/proto"
utilrand "k8s.io/apimachinery/pkg/util/rand"
"k8s.io/apimachinery/pkg/util/uuid"
genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/storage/value"
@@ -167,7 +169,9 @@ func TestEnvelopeCaching(t *testing.T) {
envelopeService := newTestEnvelopeService()
fakeClock := testingclock.NewFakeClock(time.Now())
state, err := testStateFunc(ctx, envelopeService, fakeClock)()
useSeed := randomBool()
state, err := testStateFunc(ctx, envelopeService, fakeClock, useSeed)()
if err != nil {
t.Fatal(err)
}
@@ -196,7 +200,7 @@ func TestEnvelopeCaching(t *testing.T) {
// force GC to run by performing a write
transformer.(*envelopeTransformer).cache.set([]byte("some-other-unrelated-key"), &envelopeTransformer{})
state, err = testStateFunc(ctx, envelopeService, fakeClock)()
state, err = testStateFunc(ctx, envelopeService, fakeClock, useSeed)()
if err != nil {
t.Fatal(err)
}
@@ -228,17 +232,15 @@ func TestEnvelopeCaching(t *testing.T) {
}
}
func testStateFunc(ctx context.Context, envelopeService kmsservice.Service, clock clock.Clock) func() (State, error) {
func testStateFunc(ctx context.Context, envelopeService kmsservice.Service, clock clock.Clock, useSeed bool) func() (State, error) {
return func() (State, error) {
transformer, resp, cacheKey, errGen := GenerateTransformer(ctx, string(uuid.NewUUID()), envelopeService)
transformer, encObject, cacheKey, errGen := GenerateTransformer(ctx, string(uuid.NewUUID()), envelopeService, useSeed)
if errGen != nil {
return State{}, errGen
}
return State{
Transformer: transformer,
EncryptedDEK: resp.Ciphertext,
KeyID: resp.KeyID,
Annotations: resp.Annotations,
EncryptedObject: *encObject,
UID: "panda",
ExpirationTimestamp: clock.Now().Add(time.Hour),
CacheKey: cacheKey,
@@ -254,6 +256,8 @@ func TestEnvelopeTransformerStaleness(t *testing.T) {
expectedStale bool
testErr error
testKeyID string
useSeedWrite bool
useSeedRead bool
}{
{
desc: "stateFunc returns err",
@@ -262,11 +266,35 @@ func TestEnvelopeTransformerStaleness(t *testing.T) {
testKeyID: "",
},
{
desc: "stateFunc returns same keyID",
desc: "stateFunc returns same keyID, not using seed",
expectedStale: false,
testErr: nil,
testKeyID: testKeyVersion,
},
{
desc: "stateFunc returns same keyID, using seed",
expectedStale: false,
testErr: nil,
testKeyID: testKeyVersion,
useSeedWrite: true,
useSeedRead: true,
},
{
desc: "stateFunc returns same keyID, migrating away from seed",
expectedStale: true,
testErr: nil,
testKeyID: testKeyVersion,
useSeedWrite: true,
useSeedRead: false,
},
{
desc: "stateFunc returns same keyID, migrating to seed",
expectedStale: true,
testErr: nil,
testKeyID: testKeyVersion,
useSeedWrite: false,
useSeedRead: true,
},
{
desc: "stateFunc returns different keyID",
expectedStale: true,
@@ -283,7 +311,7 @@ func TestEnvelopeTransformerStaleness(t *testing.T) {
ctx := testContext(t)
envelopeService := newTestEnvelopeService()
state, err := testStateFunc(ctx, envelopeService, clock.RealClock{})()
state, err := testStateFunc(ctx, envelopeService, clock.RealClock{}, tt.useSeedWrite)()
if err != nil {
t.Fatal(err)
}
@@ -302,7 +330,12 @@ func TestEnvelopeTransformerStaleness(t *testing.T) {
}
// inject test data before performing a read
state.KeyID = tt.testKeyID
state.EncryptedObject.KeyID = tt.testKeyID
if tt.useSeedRead {
state.EncryptedObject.EncryptedDEKSourceType = kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED
} else {
state.EncryptedObject.EncryptedDEKSourceType = kmstypes.EncryptedDEKSourceType_AES_GCM_KEY
}
stateErr = tt.testErr
_, stale, err := transformer.TransformFromStorage(ctx, transformedData, dataCtx)
@@ -330,8 +363,10 @@ func TestEnvelopeTransformerStateFunc(t *testing.T) {
ctx := testContext(t)
useSeed := randomBool()
envelopeService := newTestEnvelopeService()
state, err := testStateFunc(ctx, envelopeService, clock.RealClock{})()
state, err := testStateFunc(ctx, envelopeService, clock.RealClock{}, useSeed)()
if err != nil {
t.Fatal(err)
}
@@ -351,12 +386,18 @@ func TestEnvelopeTransformerStateFunc(t *testing.T) {
if err != stateErr {
t.Fatalf("expected state error, got: %v", err)
}
data, err := proto.Marshal(&kmstypes.EncryptedObject{
EncryptedData: []byte{1},
KeyID: "2",
EncryptedDEK: []byte{3},
Annotations: nil,
})
o := &kmstypes.EncryptedObject{
EncryptedData: []byte{1},
KeyID: "2",
EncryptedDEKSource: []byte{3},
Annotations: nil,
}
if useSeed {
o.EncryptedDEKSourceType = kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED
} else {
o.EncryptedDEKSourceType = kmstypes.EncryptedDEKSourceType_AES_GCM_KEY
}
data, err := proto.Marshal(o)
if err != nil {
t.Fatal(err)
}
@@ -401,7 +442,7 @@ func TestEnvelopeTransformerStateFunc(t *testing.T) {
t.Run("writes fail when the plugin is down and the state is invalid", func(t *testing.T) {
_, err := transformer.TransformToStorage(ctx, originalText, dataCtx)
if !strings.Contains(errString(err), `EDEK with keyID hash "sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b" expired at`) {
if !strings.Contains(errString(err), `encryptedDEKSource with keyID hash "sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b" expired at`) {
t.Fatalf("expected expiration error, got: %v", err)
}
})
@@ -418,7 +459,9 @@ func TestEnvelopeTransformerStateFunc(t *testing.T) {
if err := proto.Unmarshal(encryptedData, obj); err != nil {
t.Fatal(err)
}
obj.EncryptedDEK = append(obj.EncryptedDEK, 1) // skip StateFunc transformer
obj.EncryptedDEKSource = append(obj.EncryptedDEKSource, 1) // skip StateFunc transformer
data, err := proto.Marshal(obj)
if err != nil {
t.Fatal(err)
@@ -468,7 +511,7 @@ func TestTransformToStorageError(t *testing.T) {
envelopeService := newTestEnvelopeService()
envelopeService.SetAnnotations(tt.annotations)
transformer := NewEnvelopeTransformer(envelopeService, testProviderName,
testStateFunc(ctx, envelopeService, clock.RealClock{}),
testStateFunc(ctx, envelopeService, clock.RealClock{}, randomBool()),
)
dataCtx := value.DefaultContext(testContextText)
@@ -487,9 +530,9 @@ func TestEncodeDecode(t *testing.T) {
transformer := &envelopeTransformer{}
obj := &kmstypes.EncryptedObject{
EncryptedData: []byte{0x01, 0x02, 0x03},
KeyID: "1",
EncryptedDEK: []byte{0x04, 0x05, 0x06},
EncryptedData: []byte{0x01, 0x02, 0x03},
KeyID: "1",
EncryptedDEKSource: []byte{0x04, 0x05, 0x06},
}
data, err := transformer.doEncode(obj)
@@ -522,24 +565,62 @@ func TestValidateEncryptedObject(t *testing.T) {
{
desc: "encrypted data is nil",
originalData: &kmstypes.EncryptedObject{
KeyID: "1",
EncryptedDEK: []byte{0x01, 0x02, 0x03},
KeyID: "1",
EncryptedDEKSource: []byte{0x01, 0x02, 0x03},
},
expectedError: fmt.Errorf("encrypted data is empty"),
},
{
desc: "encrypted data is []byte{}",
originalData: &kmstypes.EncryptedObject{
EncryptedDEK: []byte{0x01, 0x02, 0x03},
EncryptedData: []byte{},
EncryptedDEKSource: []byte{0x01, 0x02, 0x03},
EncryptedData: []byte{},
},
expectedError: fmt.Errorf("encrypted data is empty"),
},
{
desc: "invalid dek source type",
originalData: &kmstypes.EncryptedObject{
EncryptedDEKSource: []byte{0x01, 0x02, 0x03},
EncryptedData: []byte{0},
EncryptedDEKSourceType: 55,
},
expectedError: fmt.Errorf("unknown encryptedDEKSourceType: 55"),
},
{
desc: "empty dek source",
originalData: &kmstypes.EncryptedObject{
EncryptedData: []byte{0},
EncryptedDEKSourceType: 1,
KeyID: "1",
},
expectedError: fmt.Errorf("failed to validate encrypted DEK source: encrypted DEK source is empty"),
},
{
desc: "empty key ID",
originalData: &kmstypes.EncryptedObject{
EncryptedDEKSource: []byte{0x01, 0x02, 0x03},
EncryptedData: []byte{0},
EncryptedDEKSourceType: 1,
},
expectedError: fmt.Errorf("failed to validate key id: keyID is empty"),
},
{
desc: "invalid annotations",
originalData: &kmstypes.EncryptedObject{
EncryptedDEKSource: []byte{0x01, 0x02, 0x03},
EncryptedData: []byte{0},
EncryptedDEKSourceType: 1,
KeyID: "1",
Annotations: map[string][]byte{"@": nil},
},
expectedError: fmt.Errorf(`failed to validate annotations: annotations: Invalid value: "@": a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*')`),
},
}
for _, tt := range testCases {
t.Run(tt.desc, func(t *testing.T) {
err := validateEncryptedObject(tt.originalData)
err := ValidateEncryptedObject(tt.originalData)
if err == nil {
t.Fatalf("envelopeTransformer: expected error while decoding data, got nil")
}
@@ -702,32 +783,32 @@ func TestValidateKeyID(t *testing.T) {
}
}
func TestValidateEncryptedDEK(t *testing.T) {
func TestValidateEncryptedDEKSource(t *testing.T) {
t.Parallel()
testCases := []struct {
name string
encryptedDEK []byte
expectedError string
name string
encryptedDEKSource []byte
expectedError string
}{
{
name: "encrypted DEK is nil",
encryptedDEK: nil,
expectedError: "encrypted DEK is empty",
name: "encrypted DEK source is nil",
encryptedDEKSource: nil,
expectedError: "encrypted DEK source is empty",
},
{
name: "encrypted DEK is empty",
encryptedDEK: []byte{},
expectedError: "encrypted DEK is empty",
name: "encrypted DEK source is empty",
encryptedDEKSource: []byte{},
expectedError: "encrypted DEK source is empty",
},
{
name: "encrypted DEK size is greater than 1 kB",
encryptedDEK: bytes.Repeat([]byte("a"), 1024+1),
expectedError: "which exceeds the max size of",
name: "encrypted DEK source size is greater than 1 kB",
encryptedDEKSource: bytes.Repeat([]byte("a"), 1024+1),
expectedError: "which exceeds the max size of",
},
{
name: "valid encrypted DEK",
encryptedDEK: []byte{0x01, 0x02, 0x03},
expectedError: "",
name: "valid encrypted DEK source",
encryptedDEKSource: []byte{0x01, 0x02, 0x03},
expectedError: "",
},
}
@@ -735,7 +816,7 @@ func TestValidateEncryptedDEK(t *testing.T) {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
err := validateEncryptedDEK(tt.encryptedDEK)
err := validateEncryptedDEKSource(tt.encryptedDEKSource)
if tt.expectedError != "" {
if err == nil {
t.Fatalf("expected error %q, got nil", tt.expectedError)
@@ -755,7 +836,7 @@ func TestValidateEncryptedDEK(t *testing.T) {
func TestEnvelopeMetrics(t *testing.T) {
envelopeService := newTestEnvelopeService()
transformer := NewEnvelopeTransformer(envelopeService, testProviderName,
testStateFunc(testContext(t), envelopeService, clock.RealClock{}),
testStateFunc(testContext(t), envelopeService, clock.RealClock{}, randomBool()),
)
dataCtx := value.DefaultContext(testContextText)
@@ -809,8 +890,12 @@ func TestEnvelopeMetrics(t *testing.T) {
}
}
var flagOnce sync.Once // support running `go test -count X`
func TestEnvelopeLogging(t *testing.T) {
klog.InitFlags(nil)
flagOnce.Do(func() {
klog.InitFlags(nil)
})
flag.Set("v", "6")
flag.Parse()
@@ -858,7 +943,7 @@ func TestEnvelopeLogging(t *testing.T) {
envelopeService := newTestEnvelopeService()
fakeClock := testingclock.NewFakeClock(time.Now())
transformer := newEnvelopeTransformerWithClock(envelopeService, testProviderName,
testStateFunc(tc.ctx, envelopeService, clock.RealClock{}),
testStateFunc(tc.ctx, envelopeService, clock.RealClock{}, randomBool()),
1*time.Second, fakeClock)
dataCtx := value.DefaultContext([]byte(testContextText))
@@ -905,7 +990,7 @@ func TestCacheNotCorrupted(t *testing.T) {
fakeClock := testingclock.NewFakeClock(time.Now())
state, err := testStateFunc(ctx, envelopeService, fakeClock)()
state, err := testStateFunc(ctx, envelopeService, fakeClock, randomBool())()
if err != nil {
t.Fatal(err)
}
@@ -923,15 +1008,15 @@ func TestCacheNotCorrupted(t *testing.T) {
}
// this is to mimic a plugin that sets a static response for ciphertext
// but uses the annotation field to send the actual encrypted DEK.
envelopeService.SetCiphertext(state.EncryptedDEK)
// but uses the annotation field to send the actual encrypted DEK source.
envelopeService.SetCiphertext(state.EncryptedObject.EncryptedDEKSource)
// for this plugin, it indicates a change in the remote key ID as the returned
// encrypted DEK is different.
// encrypted DEK source is different.
envelopeService.SetAnnotations(map[string][]byte{
"encrypted-dek.kms.kubernetes.io": []byte("encrypted-dek-1"),
})
state, err = testStateFunc(ctx, envelopeService, fakeClock)()
state, err = testStateFunc(ctx, envelopeService, fakeClock, randomBool())()
if err != nil {
t.Fatal(err)
}
@@ -954,28 +1039,40 @@ func TestCacheNotCorrupted(t *testing.T) {
}
func TestGenerateCacheKey(t *testing.T) {
encryptedDEK1 := []byte{1, 2, 3}
encryptedDEKSource1 := []byte{1, 2, 3}
keyID1 := "id1"
annotations1 := map[string][]byte{"a": {4, 5}, "b": {6, 7}}
encryptedDEKSourceType1 := kmstypes.EncryptedDEKSourceType_AES_GCM_KEY
encryptedDEK2 := []byte{4, 5, 6}
encryptedDEKSource2 := []byte{4, 5, 6}
keyID2 := "id2"
annotations2 := map[string][]byte{"x": {9, 10}, "y": {11, 12}}
encryptedDEKSourceType2 := kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED
// generate all possible combinations of the above
testCases := []struct {
encryptedDEK []byte
keyID string
annotations map[string][]byte
encryptedDEKSourceType kmstypes.EncryptedDEKSourceType
encryptedDEKSource []byte
keyID string
annotations map[string][]byte
}{
{encryptedDEK1, keyID1, annotations1},
{encryptedDEK1, keyID1, annotations2},
{encryptedDEK1, keyID2, annotations1},
{encryptedDEK1, keyID2, annotations2},
{encryptedDEK2, keyID1, annotations1},
{encryptedDEK2, keyID1, annotations2},
{encryptedDEK2, keyID2, annotations1},
{encryptedDEK2, keyID2, annotations2},
{encryptedDEKSourceType1, encryptedDEKSource1, keyID1, annotations1},
{encryptedDEKSourceType1, encryptedDEKSource1, keyID1, annotations2},
{encryptedDEKSourceType1, encryptedDEKSource1, keyID2, annotations1},
{encryptedDEKSourceType1, encryptedDEKSource1, keyID2, annotations2},
{encryptedDEKSourceType1, encryptedDEKSource2, keyID1, annotations1},
{encryptedDEKSourceType1, encryptedDEKSource2, keyID1, annotations2},
{encryptedDEKSourceType1, encryptedDEKSource2, keyID2, annotations1},
{encryptedDEKSourceType1, encryptedDEKSource2, keyID2, annotations2},
{encryptedDEKSourceType2, encryptedDEKSource1, keyID1, annotations1},
{encryptedDEKSourceType2, encryptedDEKSource1, keyID1, annotations2},
{encryptedDEKSourceType2, encryptedDEKSource1, keyID2, annotations1},
{encryptedDEKSourceType2, encryptedDEKSource1, keyID2, annotations2},
{encryptedDEKSourceType2, encryptedDEKSource2, keyID1, annotations1},
{encryptedDEKSourceType2, encryptedDEKSource2, keyID1, annotations2},
{encryptedDEKSourceType2, encryptedDEKSource2, keyID2, annotations1},
{encryptedDEKSourceType2, encryptedDEKSource2, keyID2, annotations2},
}
for _, tc := range testCases {
@@ -983,8 +1080,8 @@ func TestGenerateCacheKey(t *testing.T) {
for _, tc2 := range testCases {
tc2 := tc2
t.Run(fmt.Sprintf("%+v-%+v", tc, tc2), func(t *testing.T) {
key1, err1 := generateCacheKey(tc.encryptedDEK, tc.keyID, tc.annotations)
key2, err2 := generateCacheKey(tc2.encryptedDEK, tc2.keyID, tc2.annotations)
key1, err1 := generateCacheKey(tc.encryptedDEKSourceType, tc.encryptedDEKSource, tc.keyID, tc.annotations)
key2, err2 := generateCacheKey(tc2.encryptedDEKSourceType, tc2.encryptedDEKSource, tc2.keyID, tc2.annotations)
if err1 != nil || err2 != nil {
t.Errorf("generateCacheKey() want err=nil, got err1=%q, err2=%q", errString(err1), errString(err2))
}
@@ -1028,7 +1125,7 @@ func TestGenerateTransformer(t *testing.T) {
envelopeService.SetCiphertext([]byte{})
return envelopeService
},
expectedErr: "failed to validate encrypted DEK: encrypted DEK is empty",
expectedErr: "failed to validate encrypted DEK source: encrypted DEK source is empty",
},
{
name: "invalid annotations",
@@ -1053,7 +1150,7 @@ func TestGenerateTransformer(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
transformer, encryptResp, cacheKey, err := GenerateTransformer(testContext(t), "panda", tc.envelopeService())
transformer, encObject, cacheKey, err := GenerateTransformer(testContext(t), "panda", tc.envelopeService(), randomBool())
if tc.expectedErr == "" {
if err != nil {
t.Errorf("expected no error, got %q", errString(err))
@@ -1061,7 +1158,7 @@ func TestGenerateTransformer(t *testing.T) {
if transformer == nil {
t.Error("expected transformer, got nil")
}
if encryptResp == nil {
if encObject == nil {
t.Error("expected encrypt response, got nil")
}
if cacheKey == nil {
@@ -1083,3 +1180,5 @@ func errString(err error) string {
return err.Error()
}
func randomBool() bool { return utilrand.Int()%2 == 1 }

View File

@@ -370,6 +370,7 @@ func TestKMSOperationsMetric(t *testing.T) {
}
defer destroyService(service)
metrics.RegisterMetrics()
metrics.KMSOperationsLatencyMetric.Reset() // support running `go test -count X`
testCases := []struct {
name string

View File

@@ -36,19 +36,52 @@ var _ = math.Inf
// proto package needs to be updated.
const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
type EncryptedDEKSourceType int32
const (
// AES_GCM_KEY means that the plaintext of encryptedDEKSource is the DEK itself, with AES-GCM as the encryption algorithm.
EncryptedDEKSourceType_AES_GCM_KEY EncryptedDEKSourceType = 0
// HKDF_SHA256_XNONCE_AES_GCM_SEED means that the plaintext of encryptedDEKSource is the pseudo random key
// (referred to as the seed throughout the code) that is fed into HKDF expand. SHA256 is the hash algorithm
// and first 32 bytes of encryptedData are the info param. The first 32 bytes from the HKDF stream are used
// as the DEK with AES-GCM as the encryption algorithm.
EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED EncryptedDEKSourceType = 1
)
var EncryptedDEKSourceType_name = map[int32]string{
0: "AES_GCM_KEY",
1: "HKDF_SHA256_XNONCE_AES_GCM_SEED",
}
var EncryptedDEKSourceType_value = map[string]int32{
"AES_GCM_KEY": 0,
"HKDF_SHA256_XNONCE_AES_GCM_SEED": 1,
}
func (x EncryptedDEKSourceType) String() string {
return proto.EnumName(EncryptedDEKSourceType_name, int32(x))
}
func (EncryptedDEKSourceType) EnumDescriptor() ([]byte, []int) {
return fileDescriptor_00212fb1f9d3bf1c, []int{0}
}
// EncryptedObject is the representation of data stored in etcd after envelope encryption.
type EncryptedObject struct {
// EncryptedData is the encrypted data.
EncryptedData []byte `protobuf:"bytes,1,opt,name=encryptedData,proto3" json:"encryptedData,omitempty"`
// KeyID is the KMS key ID used for encryption operations.
KeyID string `protobuf:"bytes,2,opt,name=keyID,proto3" json:"keyID,omitempty"`
// EncryptedDEK is the encrypted DEK.
EncryptedDEK []byte `protobuf:"bytes,3,opt,name=encryptedDEK,proto3" json:"encryptedDEK,omitempty"`
// EncryptedDEKSource is the ciphertext of the source of the DEK used to encrypt the data stored in encryptedData.
// encryptedDEKSourceType defines the process of using the plaintext of this field to determine the aforementioned DEK.
EncryptedDEKSource []byte `protobuf:"bytes,3,opt,name=encryptedDEKSource,proto3" json:"encryptedDEKSource,omitempty"`
// Annotations is additional metadata that was provided by the KMS plugin.
Annotations map[string][]byte `protobuf:"bytes,4,rep,name=annotations,proto3" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
Annotations map[string][]byte `protobuf:"bytes,4,rep,name=annotations,proto3" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
// encryptedDEKSourceType defines the process of using the plaintext of encryptedDEKSource to determine the DEK.
EncryptedDEKSourceType EncryptedDEKSourceType `protobuf:"varint,5,opt,name=encryptedDEKSourceType,proto3,enum=v2.EncryptedDEKSourceType" json:"encryptedDEKSourceType,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *EncryptedObject) Reset() { *m = EncryptedObject{} }
@@ -89,9 +122,9 @@ func (m *EncryptedObject) GetKeyID() string {
return ""
}
func (m *EncryptedObject) GetEncryptedDEK() []byte {
func (m *EncryptedObject) GetEncryptedDEKSource() []byte {
if m != nil {
return m.EncryptedDEK
return m.EncryptedDEKSource
}
return nil
}
@@ -103,7 +136,15 @@ func (m *EncryptedObject) GetAnnotations() map[string][]byte {
return nil
}
func (m *EncryptedObject) GetEncryptedDEKSourceType() EncryptedDEKSourceType {
if m != nil {
return m.EncryptedDEKSourceType
}
return EncryptedDEKSourceType_AES_GCM_KEY
}
func init() {
proto.RegisterEnum("v2.EncryptedDEKSourceType", EncryptedDEKSourceType_name, EncryptedDEKSourceType_value)
proto.RegisterType((*EncryptedObject)(nil), "v2.EncryptedObject")
proto.RegisterMapType((map[string][]byte)(nil), "v2.EncryptedObject.AnnotationsEntry")
}
@@ -111,21 +152,26 @@ func init() {
func init() { proto.RegisterFile("api.proto", fileDescriptor_00212fb1f9d3bf1c) }
var fileDescriptor_00212fb1f9d3bf1c = []byte{
// 244 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x90, 0xb1, 0x4b, 0x03, 0x31,
0x14, 0xc6, 0xc9, 0x9d, 0x0a, 0x97, 0x9e, 0x58, 0x82, 0xc3, 0xe1, 0x74, 0x94, 0x0e, 0x37, 0x25,
0x10, 0x97, 0x22, 0x52, 0x50, 0x7a, 0x82, 0x38, 0x08, 0x19, 0xdd, 0xd2, 0xfa, 0x28, 0x67, 0x6a,
0x12, 0x92, 0x18, 0xc8, 0x9f, 0xee, 0x26, 0x4d, 0x95, 0xda, 0xdb, 0xde, 0xf7, 0xf1, 0xfb, 0xe0,
0xc7, 0xc3, 0x95, 0xb4, 0x03, 0xb5, 0xce, 0x04, 0x43, 0x8a, 0xc8, 0x67, 0xdf, 0x08, 0x5f, 0xf5,
0x7a, 0xe3, 0x92, 0x0d, 0xf0, 0xfe, 0xba, 0xfe, 0x80, 0x4d, 0x20, 0x73, 0x7c, 0x09, 0x7f, 0xd5,
0x4a, 0x06, 0xd9, 0xa0, 0x16, 0x75, 0xb5, 0x38, 0x2d, 0xc9, 0x35, 0x3e, 0x57, 0x90, 0x9e, 0x57,
0x4d, 0xd1, 0xa2, 0xae, 0x12, 0x87, 0x40, 0x66, 0xb8, 0x3e, 0x62, 0xfd, 0x4b, 0x53, 0xe6, 0xe9,
0x49, 0x47, 0x9e, 0xf0, 0x44, 0x6a, 0x6d, 0x82, 0x0c, 0x83, 0xd1, 0xbe, 0x39, 0x6b, 0xcb, 0x6e,
0xc2, 0xe7, 0x34, 0x72, 0x3a, 0x32, 0xa1, 0x0f, 0x47, 0xac, 0xd7, 0xc1, 0x25, 0xf1, 0x7f, 0x78,
0xb3, 0xc4, 0xd3, 0x31, 0x40, 0xa6, 0xb8, 0x54, 0x90, 0xb2, 0x71, 0x25, 0xf6, 0xe7, 0xde, 0x33,
0xca, 0xdd, 0x17, 0x64, 0xcf, 0x5a, 0x1c, 0xc2, 0x5d, 0xb1, 0x40, 0x8f, 0xcb, 0xb7, 0x7b, 0xb5,
0xf0, 0x74, 0x30, 0x4c, 0xda, 0xc1, 0x83, 0x8b, 0xe0, 0x98, 0x55, 0x5b, 0xe6, 0x83, 0x71, 0x72,
0x0b, 0x2c, 0x93, 0xec, 0x57, 0x9d, 0x81, 0x8e, 0xb0, 0x33, 0x16, 0x98, 0xfa, 0xf4, 0x91, 0xb3,
0xc8, 0xd7, 0x17, 0xf9, 0x8d, 0xb7, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x00, 0x80, 0x43, 0x93,
0x53, 0x01, 0x00, 0x00,
// 329 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x91, 0xe1, 0x4b, 0xc2, 0x40,
0x18, 0xc6, 0xdb, 0xcc, 0xc0, 0xd3, 0x72, 0x1c, 0x21, 0xc3, 0x2f, 0x8d, 0xf2, 0xc3, 0xe8, 0xc3,
0x0e, 0x16, 0x85, 0x44, 0x08, 0xe6, 0xce, 0x0c, 0x49, 0x61, 0xeb, 0x43, 0xf5, 0x65, 0x9c, 0xf6,
0x22, 0x6b, 0xb6, 0x1b, 0xb7, 0xf3, 0x60, 0x7f, 0x6a, 0xff, 0x4d, 0x38, 0x13, 0xd3, 0xec, 0xdb,
0xbd, 0xef, 0xfd, 0xde, 0xe7, 0xb9, 0x7b, 0x5e, 0x54, 0x61, 0x69, 0xe4, 0xa4, 0x82, 0x4b, 0x8e,
0x75, 0xe5, 0x9e, 0x7f, 0xe9, 0xa8, 0x4e, 0x93, 0xa9, 0xc8, 0x53, 0x09, 0xef, 0xe3, 0xc9, 0x07,
0x4c, 0x25, 0x6e, 0xa1, 0x63, 0x58, 0xb7, 0x3c, 0x26, 0x99, 0xa9, 0x59, 0x9a, 0x5d, 0xf3, 0xb7,
0x9b, 0xf8, 0x14, 0x95, 0x63, 0xc8, 0x1f, 0x3d, 0x53, 0xb7, 0x34, 0xbb, 0xe2, 0xaf, 0x0a, 0xec,
0x20, 0xbc, 0xc1, 0xe8, 0x30, 0xe0, 0x0b, 0x31, 0x05, 0xb3, 0x54, 0x08, 0xec, 0xb9, 0xc1, 0x7d,
0x54, 0x65, 0x49, 0xc2, 0x25, 0x93, 0x11, 0x4f, 0x32, 0xf3, 0xd0, 0x2a, 0xd9, 0x55, 0xb7, 0xe5,
0x28, 0xd7, 0xd9, 0x79, 0x95, 0xd3, 0xdd, 0x60, 0x34, 0x91, 0x22, 0xf7, 0x7f, 0x0f, 0x62, 0x1f,
0x35, 0xfe, 0xaa, 0x3f, 0xe7, 0x29, 0x98, 0x65, 0x4b, 0xb3, 0x4f, 0xdc, 0xe6, 0x96, 0xe4, 0x16,
0xe1, 0xff, 0x33, 0xd9, 0xec, 0x20, 0x63, 0xd7, 0x14, 0x1b, 0xa8, 0x14, 0x43, 0x5e, 0x24, 0x52,
0xf1, 0x97, 0xc7, 0x65, 0x0e, 0x8a, 0xcd, 0x17, 0x50, 0xe4, 0x50, 0xf3, 0x57, 0xc5, 0xad, 0xde,
0xd6, 0x2e, 0x47, 0xa8, 0xb1, 0xdf, 0x11, 0xd7, 0x51, 0xb5, 0x4b, 0x83, 0xf0, 0xa1, 0xf7, 0x14,
0x0e, 0xe9, 0xab, 0x71, 0x80, 0x2f, 0xd0, 0xd9, 0x60, 0xe8, 0xf5, 0xc3, 0x60, 0xd0, 0x75, 0xaf,
0x6f, 0xc2, 0x97, 0xd1, 0x78, 0xd4, 0xa3, 0xe1, 0x9a, 0x09, 0x28, 0xf5, 0x0c, 0xed, 0xbe, 0xf3,
0x76, 0x17, 0xb7, 0x33, 0x27, 0xe2, 0x84, 0xa5, 0x51, 0x06, 0x42, 0x81, 0x20, 0x69, 0x3c, 0x23,
0x99, 0xe4, 0x82, 0xcd, 0x80, 0x14, 0xce, 0xe4, 0xe7, 0x33, 0x04, 0x12, 0x05, 0x73, 0x9e, 0x02,
0x89, 0x3f, 0x33, 0xe5, 0x12, 0xe5, 0x4e, 0x8e, 0x8a, 0xb5, 0x5f, 0x7d, 0x07, 0x00, 0x00, 0xff,
0xff, 0xcc, 0x0f, 0x2b, 0x2e, 0x03, 0x02, 0x00, 0x00,
}

View File

@@ -28,9 +28,24 @@ message EncryptedObject {
// KeyID is the KMS key ID used for encryption operations.
string keyID = 2;
// EncryptedDEK is the encrypted DEK.
bytes encryptedDEK = 3;
// EncryptedDEKSource is the ciphertext of the source of the DEK used to encrypt the data stored in encryptedData.
// encryptedDEKSourceType defines the process of using the plaintext of this field to determine the aforementioned DEK.
bytes encryptedDEKSource = 3;
// Annotations is additional metadata that was provided by the KMS plugin.
map<string, bytes> annotations = 4;
// encryptedDEKSourceType defines the process of using the plaintext of encryptedDEKSource to determine the DEK.
EncryptedDEKSourceType encryptedDEKSourceType = 5;
}
enum EncryptedDEKSourceType {
// AES_GCM_KEY means that the plaintext of encryptedDEKSource is the DEK itself, with AES-GCM as the encryption algorithm.
AES_GCM_KEY = 0;
// HKDF_SHA256_XNONCE_AES_GCM_SEED means that the plaintext of encryptedDEKSource is the pseudo random key
// (referred to as the seed throughout the code) that is fed into HKDF expand. SHA256 is the hash algorithm
// and first 32 bytes of encryptedData are the info param. The first 32 bytes from the HKDF stream are used
// as the DEK with AES-GCM as the encryption algorithm.
HKDF_SHA256_XNONCE_AES_GCM_SEED = 1;
}

View File

@@ -60,7 +60,7 @@ type Base64Plugin struct {
}
// NewBase64Plugin is a constructor for Base64Plugin.
func NewBase64Plugin(t *testing.T, socketPath string) *Base64Plugin {
func NewBase64Plugin(t testing.TB, socketPath string) *Base64Plugin {
server := grpc.NewServer()
result := &Base64Plugin{
grpcServer: server,

View File

@@ -23,8 +23,10 @@ import (
"bytes"
"context"
"crypto/aes"
"crypto/cipher"
"encoding/binary"
"fmt"
"io"
"strings"
"testing"
"time"
@@ -36,11 +38,16 @@ import (
apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
utilrand "k8s.io/apimachinery/pkg/util/rand"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/features"
"k8s.io/apiserver/pkg/registry/generic"
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/server/options/encryptionconfig"
"k8s.io/apiserver/pkg/storage/storagebackend"
"k8s.io/apiserver/pkg/storage/value"
@@ -52,16 +59,23 @@ import (
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/klog/v2"
kmsv2api "k8s.io/kms/apis/v2"
kmsv2svc "k8s.io/kms/pkg/service"
"k8s.io/kubernetes/pkg/api/legacyscheme"
api "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/controlplane"
"k8s.io/kubernetes/pkg/kubeapiserver"
secretstore "k8s.io/kubernetes/pkg/registry/core/secret/storage"
"k8s.io/kubernetes/test/integration"
"k8s.io/kubernetes/test/integration/etcd"
)
type envelopekmsv2 struct {
providerName string
rawEnvelope []byte
plainTextDEK []byte
providerName string
rawEnvelope []byte
plainTextDEKSource []byte
useSeed bool
}
func (r envelopekmsv2) prefix() string {
@@ -72,12 +86,25 @@ func (r envelopekmsv2) prefixLen() int {
return len(r.prefix())
}
func (r envelopekmsv2) cipherTextDEK() ([]byte, error) {
func (r envelopekmsv2) cipherTextDEKSource() ([]byte, error) {
o := &kmstypes.EncryptedObject{}
if err := proto.Unmarshal(r.rawEnvelope[r.startOfPayload(r.providerName):], o); err != nil {
return nil, err
}
return o.EncryptedDEK, nil
if err := kmsv2.ValidateEncryptedObject(o); err != nil {
return nil, err
}
if r.useSeed && o.EncryptedDEKSourceType != kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED {
return nil, fmt.Errorf("wrong type used with useSeed=true")
}
if !r.useSeed && o.EncryptedDEKSourceType != kmstypes.EncryptedDEKSourceType_AES_GCM_KEY {
return nil, fmt.Errorf("wrong type used with useSeed=false")
}
return o.EncryptedDEKSource, nil
}
func (r envelopekmsv2) startOfPayload(_ string) int {
@@ -89,25 +116,39 @@ func (r envelopekmsv2) cipherTextPayload() ([]byte, error) {
if err := proto.Unmarshal(r.rawEnvelope[r.startOfPayload(r.providerName):], o); err != nil {
return nil, err
}
if err := kmsv2.ValidateEncryptedObject(o); err != nil {
return nil, err
}
return o.EncryptedData, nil
}
func (r envelopekmsv2) plainTextPayload(secretETCDPath string) ([]byte, error) {
block, err := aes.NewCipher(r.plainTextDEK)
if err != nil {
return nil, fmt.Errorf("failed to initialize AES Cipher: %v", err)
var transformer value.Read
var err error
if r.useSeed {
transformer, err = aestransformer.NewHKDFExtendedNonceGCMTransformer(r.plainTextDEKSource)
} else {
var block cipher.Block
block, err = aes.NewCipher(r.plainTextDEKSource)
if err != nil {
return nil, err
}
transformer, err = aestransformer.NewGCMTransformer(block)
}
if err != nil {
return nil, err
}
ctx := context.Background()
dataCtx := value.DefaultContext(secretETCDPath)
aesgcmTransformer, err := aestransformer.NewGCMTransformer(block)
if err != nil {
return nil, fmt.Errorf("failed to create transformer from block: %v", err)
}
data, err := r.cipherTextPayload()
if err != nil {
return nil, fmt.Errorf("failed to get cipher text payload: %v", err)
}
plainSecret, _, err := aesgcmTransformer.TransformFromStorage(ctx, data, dataCtx)
plainSecret, _, err := transformer.TransformFromStorage(ctx, data, dataCtx)
if err != nil {
return nil, fmt.Errorf("failed to transform from storage via AESGCM, err: %w", err)
}
@@ -118,11 +159,23 @@ func (r envelopekmsv2) plainTextPayload(secretETCDPath string) ([]byte, error) {
// TestKMSv2Provider is an integration test between KubeAPI, ETCD and KMSv2 Plugin
// Concretely, this test verifies the following integration contracts:
// 1. Raw records in ETCD that were processed by KMSv2 Provider should be prefixed with k8s:enc:kms:v2:<plugin name>:
// 2. Data Encryption Key (DEK) should be generated by envelopeTransformer and passed to KMS gRPC Plugin
// 3. KMS gRPC Plugin should encrypt the DEK with a Key Encryption Key (KEK) and pass it back to envelopeTransformer
// 4. The cipherTextPayload (ex. Secret) should be encrypted via AES GCM transform
// 2. Data Encryption Key (DEK) / DEK seed should be generated by envelopeTransformer and passed to KMS gRPC Plugin
// 3. KMS gRPC Plugin should encrypt the DEK/seed with a Key Encryption Key (KEK) and pass it back to envelopeTransformer
// 4. The cipherTextPayload (ex. Secret) should be encrypted via AES GCM transform / extended nonce GCM
// 5. kmstypes.EncryptedObject structure should be serialized and deposited in ETCD
func TestKMSv2Provider(t *testing.T) {
t.Run("regular gcm", func(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, false)()
testKMSv2Provider(t)
})
t.Run("extended nonce gcm", func(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, true)()
testKMSv2Provider(t)
})
}
func testKMSv2Provider(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)()
encryptionConfig := `
@@ -152,8 +205,7 @@ resources:
t.Fatalf("Failed to create test secret, error: %v", err)
}
// Since Data Encryption Key (DEK) is randomly generated (per encryption operation), we need to ask KMS Mock for it.
plainTextDEK := pluginMock.LastEncryptRequest()
plainTextDEKSource := pluginMock.LastEncryptRequest()
secretETCDPath := test.getETCDPathForResource(test.storageConfig.Prefix, "", "secrets", test.secret.Name, test.secret.Namespace)
rawEnvelope, err := test.getRawSecretFromETCD()
@@ -162,9 +214,10 @@ resources:
}
envelopeData := envelopekmsv2{
providerName: providerName,
rawEnvelope: rawEnvelope,
plainTextDEK: plainTextDEK,
providerName: providerName,
rawEnvelope: rawEnvelope,
plainTextDEKSource: plainTextDEKSource,
useSeed: utilfeature.DefaultFeatureGate.Enabled(features.KMSv2KDF),
}
wantPrefix := envelopeData.prefix()
@@ -174,19 +227,19 @@ resources:
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
ciphertext, err := envelopeData.cipherTextDEK()
ciphertext, err := envelopeData.cipherTextDEKSource()
if err != nil {
t.Fatalf("failed to get ciphertext DEK from KMSv2 Plugin: %v", err)
t.Fatalf("failed to get ciphertext DEK/seed from KMSv2 Plugin: %v", err)
}
decryptResponse, err := pluginMock.Decrypt(ctx, &kmsv2api.DecryptRequest{Uid: string(uuid.NewUUID()), Ciphertext: ciphertext})
if err != nil {
t.Fatalf("failed to decrypt DEK, %v", err)
}
dekPlainAsWouldBeSeenByETCD := decryptResponse.Plaintext
dekSourcePlainAsWouldBeSeenByETCD := decryptResponse.Plaintext
if !bytes.Equal(plainTextDEK, dekPlainAsWouldBeSeenByETCD) {
t.Fatalf("expected plainTextDEK %v to be passed to KMS Plugin, but got %s",
plainTextDEK, dekPlainAsWouldBeSeenByETCD)
if !bytes.Equal(plainTextDEKSource, dekSourcePlainAsWouldBeSeenByETCD) {
t.Fatalf("expected plainTextDEKSource %v to be passed to KMS Plugin, but got %s",
plainTextDEKSource, dekSourcePlainAsWouldBeSeenByETCD)
}
plainSecret, err := envelopeData.plainTextPayload(secretETCDPath)
@@ -214,11 +267,23 @@ resources:
// 1. When the key ID is unchanged, the resource version must not change
// 2. When the key ID changes, the resource version changes (but only once)
// 3. For all subsequent updates, the resource version must not change
// 4. When kms-plugin is down, expect creation of new pod and encryption to succeed while the DEK is valid
// 5. when kms-plugin is down, no-op update for a pod should succeed and not result in RV change while the DEK is valid
// 6. When kms-plugin is down, expect creation of new pod and encryption to fail once the DEK is invalid
// 7. when kms-plugin is down, no-op update for a pod should succeed and not result in RV change even once the DEK is valid
// 4. When kms-plugin is down, expect creation of new pod and encryption to succeed while the DEK/seed is valid
// 5. when kms-plugin is down, no-op update for a pod should succeed and not result in RV change while the DEK/seed is valid
// 6. When kms-plugin is down, expect creation of new pod and encryption to fail once the DEK/seed is invalid
// 7. when kms-plugin is down, no-op update for a pod should succeed and not result in RV change even once the DEK/seed is valid
func TestKMSv2ProviderKeyIDStaleness(t *testing.T) {
t.Run("regular gcm", func(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, false)()
testKMSv2ProviderKeyIDStaleness(t)
})
t.Run("extended nonce gcm", func(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, true)()
testKMSv2ProviderKeyIDStaleness(t)
})
}
func testKMSv2ProviderKeyIDStaleness(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)()
encryptionConfig := `
@@ -227,6 +292,7 @@ apiVersion: apiserver.config.k8s.io/v1
resources:
- resources:
- pods
- deployments.apps
providers:
- kms:
apiVersion: v2
@@ -262,12 +328,21 @@ resources:
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
t.Cleanup(cancel)
var firstEncryptedDEK []byte
assertPodDEKs(ctx, t, test.kubeAPIServer.ServerOpts.Etcd.StorageConfig,
1, 1,
"k8s:enc:kms:v2:kms-provider:",
func(_ int, counter uint64, etcdKey string, obj kmstypes.EncryptedObject) {
firstEncryptedDEK = obj.EncryptedDEK
useSeed := utilfeature.DefaultFeatureGate.Enabled(features.KMSv2KDF)
var firstEncryptedDEKSource []byte
var f checkFunc
if useSeed {
f = func(_ int, _ uint64, etcdKey string, obj kmstypes.EncryptedObject) {
firstEncryptedDEKSource = obj.EncryptedDEKSource
if obj.KeyID != "1" {
t.Errorf("key %s: want key ID %s, got %s", etcdKey, "1", obj.KeyID)
}
}
} else {
f = func(_ int, counter uint64, etcdKey string, obj kmstypes.EncryptedObject) {
firstEncryptedDEKSource = obj.EncryptedDEKSource
if obj.KeyID != "1" {
t.Errorf("key %s: want key ID %s, got %s", etcdKey, "1", obj.KeyID)
@@ -279,8 +354,14 @@ resources:
if want != counter {
t.Errorf("key %s: counter nonce is invalid: want %d, got %d", etcdKey, want, counter)
}
},
}
}
assertPodDEKSources(ctx, t, test.kubeAPIServer.ServerOpts.Etcd.StorageConfig,
1, 1, "k8s:enc:kms:v2:kms-provider:", f,
)
if len(firstEncryptedDEKSource) == 0 {
t.Fatal("unexpected empty DEK or seed")
}
// 2. no-op update for the test pod with keyID update should result in RV change
pluginMock.UpdateKeyID()
@@ -318,17 +399,38 @@ resources:
// - in place update with RV change
// - delete (which does an update to set deletion timestamp)
// - create
checkDEK := func(_ int, counter uint64, etcdKey string, obj kmstypes.EncryptedObject) {
if bytes.Equal(obj.EncryptedDEK, firstEncryptedDEK) {
t.Errorf("key %s: incorrectly has the same EDEK", etcdKey)
}
var checkDEK checkFunc
if useSeed {
checkDEK = func(_ int, _ uint64, etcdKey string, obj kmstypes.EncryptedObject) {
if len(obj.EncryptedDEKSource) == 0 {
t.Error("unexpected empty DEK source")
}
if obj.KeyID != "2" {
t.Errorf("key %s: want key ID %s, got %s", etcdKey, "2", obj.KeyID)
}
if bytes.Equal(obj.EncryptedDEKSource, firstEncryptedDEKSource) {
t.Errorf("key %s: incorrectly has the same ESEED", etcdKey)
}
if wantCount != counter {
t.Errorf("key %s: counter nonce is invalid: want %d, got %d", etcdKey, wantCount, counter)
if obj.KeyID != "2" {
t.Errorf("key %s: want key ID %s, got %s", etcdKey, "2", obj.KeyID)
}
}
} else {
checkDEK = func(_ int, counter uint64, etcdKey string, obj kmstypes.EncryptedObject) {
if len(obj.EncryptedDEKSource) == 0 {
t.Error("unexpected empty DEK source")
}
if bytes.Equal(obj.EncryptedDEKSource, firstEncryptedDEKSource) {
t.Errorf("key %s: incorrectly has the same EDEK", etcdKey)
}
if obj.KeyID != "2" {
t.Errorf("key %s: want key ID %s, got %s", etcdKey, "2", obj.KeyID)
}
if wantCount != counter {
t.Errorf("key %s: counter nonce is invalid: want %d, got %d", etcdKey, wantCount, counter)
}
}
}
@@ -378,7 +480,7 @@ resources:
// 6. when kms-plugin is down, expect creation of new pod and encryption to fail because the DEK is invalid
_, err = test.createPod(testNamespace, dynamicClient)
if err == nil || !strings.Contains(err.Error(), `EDEK with keyID hash "sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35" expired at 2`) {
if err == nil || !strings.Contains(err.Error(), `encryptedDEKSource with keyID hash "sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35" expired at 2`) {
t.Fatalf("Create test pod should have failed due to encryption, ns: %s, got: %v", testNamespace, err)
}
@@ -392,12 +494,90 @@ resources:
t.Fatalf("Resource version should not have changed again after the initial version updated as a result of the keyID update. old pod: %v, new pod: %v", newPod, updatedNewPod)
}
assertPodDEKs(ctx, t, test.kubeAPIServer.ServerOpts.Etcd.StorageConfig,
assertPodDEKSources(ctx, t, test.kubeAPIServer.ServerOpts.Etcd.StorageConfig,
1, 1, "k8s:enc:kms:v2:kms-provider:", checkDEK,
)
// fix plugin and wait for new writes to start working again
kmsv2.NowFunc = origNowFunc
pluginMock.ExitFailedState()
err = wait.Poll(time.Second, 3*time.Minute,
func() (bool, error) {
t.Log("polling for plugin to be functional")
_, err = test.createDeployment("panda", testNamespace)
if err != nil {
t.Logf("failed to create deployment, plugin is likely still unhealthy: %v", err)
}
return err == nil, nil
})
if err != nil {
t.Fatalf("failed to restore plugin health, err: %v, ns: %s", err, testNamespace)
}
// 8. confirm that no-op update for a pod succeeds and still does not result in RV change
updatedNewPod2, err := test.inplaceUpdatePod(testNamespace, updatedNewPod, dynamicClient)
if err != nil {
t.Fatalf("Failed to perform no-op update on pod when kms-plugin is up, error: %v, ns: %s", err, testNamespace)
}
version8 := updatedNewPod2.GetResourceVersion()
if version7 != version8 {
t.Fatalf("Resource version should not have changed after plugin health is restored. old pod: %v, new pod: %v", updatedNewPod, updatedNewPod2)
}
// flip the current config
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, !useSeed)()
// 9. confirm that no-op update for a pod results in RV change due to KDF config change
var version9 string
err = wait.Poll(time.Second, 3*time.Minute,
func() (bool, error) {
t.Log("polling for in-place update rv change due to KDF config change")
updatedNewPod2, err = test.inplaceUpdatePod(testNamespace, updatedNewPod2, dynamicClient)
if err != nil {
return false, err
}
version9 = updatedNewPod2.GetResourceVersion()
if version8 != version9 {
return true, nil
}
return false, nil
})
if err != nil {
t.Fatalf("Failed to detect one resource version update within the allotted time after KDF config change and pod has been inplace updated, err: %v, ns: %s", err, testNamespace)
}
}
func TestKMSv2ProviderDEKReuse(t *testing.T) {
func TestKMSv2ProviderDEKSourceReuse(t *testing.T) {
t.Run("regular gcm", func(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, false)()
testKMSv2ProviderDEKSourceReuse(t,
func(i int, counter uint64, etcdKey string, obj kmstypes.EncryptedObject) {
if obj.KeyID != "1" {
t.Errorf("key %s: want key ID %s, got %s", etcdKey, "1", obj.KeyID)
}
// zero value of counter is one billion so the first value will be one billion plus one
// hence we add that to our zero based index to calculate the expected nonce
if uint64(i+1_000_000_000+1) != counter {
t.Errorf("key %s: counter nonce is invalid: want %d, got %d", etcdKey, i+1, counter)
}
},
)
})
t.Run("extended nonce gcm", func(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, true)()
testKMSv2ProviderDEKSourceReuse(t,
func(_ int, _ uint64, etcdKey string, obj kmstypes.EncryptedObject) {
if obj.KeyID != "1" {
t.Errorf("key %s: want key ID %s, got %s", etcdKey, "1", obj.KeyID)
}
},
)
})
}
func testKMSv2ProviderDEKSourceReuse(t *testing.T, f checkFunc) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)()
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
@@ -445,25 +625,15 @@ resources:
}
}
assertPodDEKs(ctx, t, test.kubeAPIServer.ServerOpts.Etcd.StorageConfig,
assertPodDEKSources(ctx, t, test.kubeAPIServer.ServerOpts.Etcd.StorageConfig,
podCount, 1, // key ID does not change during the test so we should only have a single DEK
"k8s:enc:kms:v2:kms-provider:",
func(i int, counter uint64, etcdKey string, obj kmstypes.EncryptedObject) {
if obj.KeyID != "1" {
t.Errorf("key %s: want key ID %s, got %s", etcdKey, "1", obj.KeyID)
}
// zero value of counter is one billion so the first value will be one billion plus one
// hence we add that to our zero based index to calculate the expected nonce
if uint64(i+1_000_000_000+1) != counter {
t.Errorf("key %s: counter nonce is invalid: want %d, got %d", etcdKey, i+1, counter)
}
},
"k8s:enc:kms:v2:kms-provider:", f,
)
}
func assertPodDEKs(ctx context.Context, t *testing.T, config storagebackend.Config, podCount, dekCount int, kmsPrefix string,
f func(i int, counter uint64, etcdKey string, obj kmstypes.EncryptedObject)) {
type checkFunc func(i int, counter uint64, etcdKey string, obj kmstypes.EncryptedObject)
func assertPodDEKSources(ctx context.Context, t *testing.T, config storagebackend.Config, podCount, dekSourcesCount int, kmsPrefix string, f checkFunc) {
t.Helper()
rawClient, etcdClient, err := integration.GetEtcdClients(config.Transport)
@@ -481,6 +651,8 @@ func assertPodDEKs(ctx context.Context, t *testing.T, config storagebackend.Conf
t.Fatalf("expected %d KVs, but got %d", podCount, len(response.Kvs))
}
useSeed := utilfeature.DefaultFeatureGate.Enabled(features.KMSv2KDF)
out := make([]kmstypes.EncryptedObject, len(response.Kvs))
for i, kv := range response.Kvs {
v := bytes.TrimPrefix(kv.Value, []byte(kmsPrefix))
@@ -488,7 +660,17 @@ func assertPodDEKs(ctx context.Context, t *testing.T, config storagebackend.Conf
t.Fatal(err)
}
nonce := out[i].EncryptedData[:12]
if err := kmsv2.ValidateEncryptedObject(&out[i]); err != nil {
t.Fatal(err)
}
var infoLen int
if useSeed {
infoLen = 32
}
info := out[i].EncryptedData[:infoLen]
nonce := out[i].EncryptedData[infoLen : 12+infoLen]
randN := nonce[:4]
count := nonce[4:]
@@ -496,22 +678,43 @@ func assertPodDEKs(ctx context.Context, t *testing.T, config storagebackend.Conf
t.Errorf("key %s: got all zeros for first four bytes", string(kv.Key))
}
if useSeed {
if bytes.Equal(info, make([]byte, infoLen)) {
t.Errorf("key %s: got all zeros for info", string(kv.Key))
}
}
counter := binary.LittleEndian.Uint64(count)
f(i, counter, string(kv.Key), out[i])
}
uniqueDEKs := sets.NewString()
uniqueDEKSources := sets.NewString()
for _, object := range out {
uniqueDEKs.Insert(string(object.EncryptedDEK))
object := object
uniqueDEKSources.Insert(string(object.EncryptedDEKSource))
if useSeed {
if object.EncryptedDEKSourceType != kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED {
t.Errorf("invalid type: %d", object.EncryptedDEKSourceType)
}
} else {
if object.EncryptedDEKSourceType != kmstypes.EncryptedDEKSourceType_AES_GCM_KEY {
t.Errorf("invalid type: %d", object.EncryptedDEKSourceType)
}
}
}
if uniqueDEKs.Len() != dekCount {
t.Errorf("expected %d DEKs, got: %d", dekCount, uniqueDEKs.Len())
if uniqueDEKSources.Has("") {
t.Error("unexpected empty DEK source seen")
}
if uniqueDEKSources.Len() != dekSourcesCount {
t.Errorf("expected %d DEK sources, got: %d", dekSourcesCount, uniqueDEKSources.Len())
}
}
func TestKMSv2Healthz(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)()
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, randomBool())()
encryptionConfig := `
kind: EncryptionConfiguration
@@ -577,6 +780,7 @@ resources:
func TestKMSv2SingleService(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)()
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, randomBool())()
var kmsv2Calls int
origEnvelopeKMSv2ServiceFactory := encryptionconfig.EnvelopeKMSv2ServiceFactory
@@ -642,3 +846,250 @@ resources:
t.Fatalf("expected a single call to KMS v2 service factory: %v", kmsv2Calls)
}
}
var benchSecret *api.Secret
func BenchmarkKMSv2KDF(b *testing.B) {
b.StopTimer()
klog.SetOutput(io.Discard)
klog.LogToStderr(false)
defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.KMSv2, true)()
defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.KMSv2KDF, false)()
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
b.Cleanup(cancel)
ctx = request.WithNamespace(ctx, testNamespace)
encryptionConfig := `
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
- resources:
- secrets
providers:
- kms:
apiVersion: v2
name: kms-provider
endpoint: unix:///@kms-provider.sock
`
_ = kmsv2mock.NewBase64Plugin(b, "@kms-provider.sock")
test, err := newTransformTest(b, encryptionConfig, false, "")
if err != nil {
b.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
}
b.Cleanup(test.cleanUp)
client := kubernetes.NewForConfigOrDie(test.kubeAPIServer.ClientConfig)
restOptionsGetter := getRESTOptionsGetterForSecrets(b, test)
secretStorage, err := secretstore.NewREST(restOptionsGetter)
if err != nil {
b.Fatal(err)
}
b.Cleanup(secretStorage.Destroy)
const dataLen = 1_000
secrets := make([]*api.Secret, dataLen)
for i := 0; i < dataLen; i++ {
secrets[i] = &api.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("test-secret-%d", i),
Namespace: testNamespace,
},
Data: map[string][]byte{
"lots_of_data": bytes.Repeat([]byte{1, 3, 3, 7}, i*dataLen/4),
},
}
}
b.StartTimer()
for i := 0; i < b.N; i++ {
_, err := secretStorage.DeleteCollection(ctx, noValidation, &metav1.DeleteOptions{}, nil)
if err != nil {
b.Fatal(err)
}
for i := 0; i < dataLen; i++ {
out, err := secretStorage.Create(ctx, secrets[i], noValidation, &metav1.CreateOptions{})
if err != nil {
b.Fatal(err)
}
benchSecret = out.(*api.Secret)
out, err = secretStorage.Get(ctx, benchSecret.Name, &metav1.GetOptions{})
if err != nil {
b.Fatal(err)
}
benchSecret = out.(*api.Secret)
}
}
b.StopTimer()
secretList, err := client.CoreV1().Secrets(testNamespace).List(ctx, metav1.ListOptions{})
if err != nil {
b.Fatal(err)
}
if secretLen := len(secretList.Items); secretLen != dataLen {
b.Errorf("unexpected secret len: want %d, got %d", dataLen, secretLen)
}
}
func getRESTOptionsGetterForSecrets(t testing.TB, test *transformTest) generic.RESTOptionsGetter {
t.Helper()
s := test.kubeAPIServer.ServerOpts
etcdConfigCopy := *s.Etcd
etcdConfigCopy.SkipHealthEndpoints = true // avoid running health check go routines
etcdConfigCopy.EncryptionProviderConfigAutomaticReload = true // hack to use DynamicTransformers in t.Cleanup below
// mostly copied from BuildGenericConfig
genericConfig := genericapiserver.NewConfig(legacyscheme.Codecs)
genericConfig.MergedResourceConfig = controlplane.DefaultAPIResourceConfigSource()
if err := s.APIEnablement.ApplyTo(genericConfig, controlplane.DefaultAPIResourceConfigSource(), legacyscheme.Scheme); err != nil {
t.Fatal(err)
}
storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig()
storageFactoryConfig.APIResourceConfig = genericConfig.MergedResourceConfig
storageFactory, err := storageFactoryConfig.Complete(&etcdConfigCopy).New()
if err != nil {
t.Fatal(err)
}
if err := etcdConfigCopy.ApplyWithStorageFactoryTo(storageFactory, genericConfig); err != nil {
t.Fatal(err)
}
transformers, ok := genericConfig.ResourceTransformers.(*encryptionconfig.DynamicTransformers)
if !ok {
t.Fatalf("incorrect type for ResourceTransformers: %T", genericConfig.ResourceTransformers)
}
t.Cleanup(func() {
// this is a hack to cause the existing transformers to shutdown
transformers.Set(nil, nil, nil, 0)
time.Sleep(10 * time.Second) // block this cleanup for longer than kmsCloseGracePeriod
})
if genericConfig.RESTOptionsGetter == nil {
t.Fatal("not REST options found")
}
opts, err := genericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: "secrets"})
if err != nil {
t.Fatal(err)
}
if err := runtime.CheckCodec(opts.StorageConfig.Codec, &api.Secret{},
schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Secret"}); err != nil {
t.Fatal(err)
}
return genericConfig.RESTOptionsGetter
}
func noValidation(_ context.Context, _ runtime.Object) error { return nil }
var benchRESTSecret *corev1.Secret
func BenchmarkKMSv2REST(b *testing.B) {
b.StopTimer()
klog.SetOutput(io.Discard)
klog.LogToStderr(false)
defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.KMSv2, true)()
defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.KMSv2KDF, false)()
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
b.Cleanup(cancel)
encryptionConfig := `
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
- resources:
- secrets
providers:
- kms:
apiVersion: v2
name: kms-provider
endpoint: unix:///@kms-provider.sock
`
_ = kmsv2mock.NewBase64Plugin(b, "@kms-provider.sock")
test, err := newTransformTest(b, encryptionConfig, false, "")
if err != nil {
b.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
}
b.Cleanup(test.cleanUp)
client := kubernetes.NewForConfigOrDie(test.kubeAPIServer.ClientConfig)
const dataLen = 1_000
secretStorage := client.CoreV1().Secrets(testNamespace)
secrets := make([]*corev1.Secret, dataLen)
for i := 0; i < dataLen; i++ {
secrets[i] = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("test-secret-%d", i),
Namespace: testNamespace,
},
Data: map[string][]byte{
"lots_of_data": bytes.Repeat([]byte{1, 3, 3, 7}, i*dataLen/4),
},
}
}
b.StartTimer()
for i := 0; i < b.N; i++ {
err := secretStorage.DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{})
if err != nil {
b.Fatal(err)
}
for i := 0; i < dataLen; i++ {
out, err := secretStorage.Create(ctx, secrets[i], metav1.CreateOptions{})
if err != nil {
b.Fatal(err)
}
benchRESTSecret = out
out, err = secretStorage.Get(ctx, benchRESTSecret.Name, metav1.GetOptions{})
if err != nil {
b.Fatal(err)
}
benchRESTSecret = out
}
}
b.StopTimer()
secretList, err := client.CoreV1().Secrets(testNamespace).List(ctx, metav1.ListOptions{})
if err != nil {
b.Fatal(err)
}
if secretLen := len(secretList.Items); secretLen != dataLen {
b.Errorf("unexpected secret len: want %d, got %d", dataLen, secretLen)
}
}
func randomBool() bool { return utilrand.Int()%2 == 1 }

View File

@@ -24,6 +24,7 @@ import (
"net"
"os"
"os/exec"
"strconv"
"strings"
"syscall"
"testing"
@@ -31,8 +32,8 @@ import (
"go.uber.org/goleak"
"google.golang.org/grpc/grpclog"
"k8s.io/klog/v2"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/util/env"
)
@@ -127,6 +128,8 @@ func RunCustomEtcd(dataDir string, customFlags []string, output io.Writer) (url
"http://127.0.0.1:0",
"-log-level",
"warn", // set to info or debug for more logs
"--quota-backend-bytes",
strconv.FormatInt(8*1024*1024*1024, 10),
}
args = append(args, customFlags...)
cmd := exec.CommandContext(ctx, etcdPath, args...)

93
vendor/golang.org/x/crypto/hkdf/hkdf.go generated vendored Normal file
View File

@@ -0,0 +1,93 @@
// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package hkdf implements the HMAC-based Extract-and-Expand Key Derivation
// Function (HKDF) as defined in RFC 5869.
//
// HKDF is a cryptographic key derivation function (KDF) with the goal of
// expanding limited input keying material into one or more cryptographically
// strong secret keys.
package hkdf // import "golang.org/x/crypto/hkdf"
import (
"crypto/hmac"
"errors"
"hash"
"io"
)
// Extract generates a pseudorandom key for use with Expand from an input secret
// and an optional independent salt.
//
// Only use this function if you need to reuse the extracted key with multiple
// Expand invocations and different context values. Most common scenarios,
// including the generation of multiple keys, should use New instead.
func Extract(hash func() hash.Hash, secret, salt []byte) []byte {
if salt == nil {
salt = make([]byte, hash().Size())
}
extractor := hmac.New(hash, salt)
extractor.Write(secret)
return extractor.Sum(nil)
}
type hkdf struct {
expander hash.Hash
size int
info []byte
counter byte
prev []byte
buf []byte
}
func (f *hkdf) Read(p []byte) (int, error) {
// Check whether enough data can be generated
need := len(p)
remains := len(f.buf) + int(255-f.counter+1)*f.size
if remains < need {
return 0, errors.New("hkdf: entropy limit reached")
}
// Read any leftover from the buffer
n := copy(p, f.buf)
p = p[n:]
// Fill the rest of the buffer
for len(p) > 0 {
f.expander.Reset()
f.expander.Write(f.prev)
f.expander.Write(f.info)
f.expander.Write([]byte{f.counter})
f.prev = f.expander.Sum(f.prev[:0])
f.counter++
// Copy the new batch into p
f.buf = f.prev
n = copy(p, f.buf)
p = p[n:]
}
// Save leftovers for next run
f.buf = f.buf[n:]
return need, nil
}
// Expand returns a Reader, from which keys can be read, using the given
// pseudorandom key and optional context info, skipping the extraction step.
//
// The pseudorandomKey should have been generated by Extract, or be a uniformly
// random or pseudorandom cryptographically strong key. See RFC 5869, Section
// 3.3. Most common scenarios will want to use New instead.
func Expand(hash func() hash.Hash, pseudorandomKey, info []byte) io.Reader {
expander := hmac.New(hash, pseudorandomKey)
return &hkdf{expander, expander.Size(), info, 1, nil, nil}
}
// New returns a Reader, from which keys can be read, using the given hash,
// secret, salt and context info. Salt and info can be nil.
func New(hash func() hash.Hash, secret, salt, info []byte) io.Reader {
prk := Extract(hash, secret, salt)
return Expand(hash, prk, info)
}

1
vendor/modules.txt vendored
View File

@@ -949,6 +949,7 @@ golang.org/x/crypto/cryptobyte/asn1
golang.org/x/crypto/curve25519
golang.org/x/crypto/curve25519/internal/field
golang.org/x/crypto/ed25519
golang.org/x/crypto/hkdf
golang.org/x/crypto/internal/alias
golang.org/x/crypto/internal/poly1305
golang.org/x/crypto/nacl/secretbox