public/k3s
1
0
Fork 0
mirror of https://git.vectorsigma.ru/public/k3s.git synced 2026-07-28 21:39:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement secrets-encryption secretbox provider

Browse Source 
- Add testlet for new provider switch
- Handle migration between providers
- Add exception for criticalcontrolargs
Signed-off-by: Derek Nola <derek.nola@suse.com>
This commit is contained in:
Derek Nola
2023-03-17 15:30:40 -07:00
parent a897f6875e
commit aea3703f68
10 changed files with 317 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
pkg/cli/cmds/secrets_encrypt.go
View File

@@ -40,8 +40,9 @@ func NewSecretsEncryptCommands(status, enable, disable, prepare, rotate, reencry
Flags: append(EncryptFlags, &cli.StringFlag{
Name: "output",
Aliases: []string{"o"},
Usage: "Status format. Default: text. Optional: json",
Usage: "Status format. Options: text, json",
Destination: &ServerConfig.EncryptOutput,
Value: "text",
}),
},
{

7
pkg/cli/cmds/server.go
View File

@@ -84,6 +84,7 @@ type Server struct {
EncryptForce bool
EncryptOutput string
EncryptSkip bool
EncryptProvider string
SystemDefaultRegistry string
StartupHooks []StartupHook
SupervisorMetrics bool
@@ -603,6 +604,12 @@ var ServerFlags = []cli.Flag{
Usage: "(experimental) Run rootless",
Destination: &ServerConfig.Rootless,
},
&cli.StringFlag{
Name: "secrets-encryption-provider",
Usage: "(experimental) Secret encryption provider (valid values: 'aescbc', 'secretbox')",
Destination: &ServerConfig.EncryptProvider,
Value: "aescbc",
},
PreferBundledBin,
SELinuxFlag,
LBServerPortFlag,

10
pkg/cli/secretsencrypt/secrets_encrypt.go
View File

@@ -136,10 +136,12 @@ func Status(app *cli.Context) error {
fmt.Fprintf(w, "Active\tKey Type\tName\n")
fmt.Fprintf(w, "------\t--------\t----\n")
if status.ActiveKey != "" {
fmt.Fprintf(w, " *\t%s\t%s\n", "AES-CBC", status.ActiveKey)
ak := strings.Split(status.ActiveKey, " ")
fmt.Fprintf(w, " *\t%s\t%s\n", ak[0], ak[1])
}
for _, k := range status.InactiveKeys {
fmt.Fprintf(w, "\t%s\t%s\n", "AES-CBC", k)
ik := strings.Split(k, " ")
fmt.Fprintf(w, "\t%s\t%s\n", ik[0], ik[1])
}
w.Flush()
fmt.Println(statusOutput + tabBuffer.String())
@@ -227,10 +229,10 @@ func RotateKeys(app *cli.Context) error {
if err != nil {
return err
}
timeout := 70 * time.Second
timeout := 90 * time.Second
if err = info.Put("/v1-"+version.Program+"/encrypt/config", b, clientaccess.WithTimeout(timeout)); err != nil {
return wrapServerError(err)
}
fmt.Println("keys rotated, reencryption started")
fmt.Println("keys rotated, reencryption finished")
return nil
}

1
pkg/cli/server/server.go
View File

@@ -176,6 +176,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
serverConfig.ControlConfig.EmbeddedRegistry = cfg.EmbeddedRegistry
serverConfig.ControlConfig.ClusterInit = cfg.ClusterInit
serverConfig.ControlConfig.EncryptSecrets = cfg.EncryptSecrets
serverConfig.ControlConfig.EncryptProvider = cfg.EncryptProvider
serverConfig.ControlConfig.EtcdExposeMetrics = cfg.EtcdExposeMetrics
serverConfig.ControlConfig.EtcdDisableSnapshots = cfg.EtcdDisableSnapshots
serverConfig.ControlConfig.SupervisorMetrics = cfg.SupervisorMetrics

4
pkg/cluster/bootstrap.go
View File

@@ -507,6 +507,10 @@ func (c *Cluster) compareConfig() error {
if clusterControl.CriticalControlArgs.EgressSelectorMode == "" {
clusterControl.CriticalControlArgs.EgressSelectorMode = c.config.CriticalControlArgs.EgressSelectorMode
}
// If the remote server is down-level, for secrets-encryption-key-type
if clusterControl.CriticalControlArgs.EncryptProvider == "" {
clusterControl.CriticalControlArgs.EncryptProvider = c.config.CriticalControlArgs.EncryptProvider
}
if diff := deep.Equal(c.config.CriticalControlArgs, clusterControl.CriticalControlArgs); diff != nil {
rc := reflect.ValueOf(clusterControl.CriticalControlArgs).Type()

1
pkg/daemons/config/types.go
View File

@@ -177,6 +177,7 @@ type CriticalControlArgs struct {
DisableNPC bool `cli:"disable-network-policy"`
DisableServiceLB bool `cli:"disable-service-lb"`
EncryptSecrets bool `cli:"secrets-encryption"`
EncryptProvider string `cli:"secrets-encryption-provider"`
EmbeddedRegistry bool `cli:"embedded-registry"`
FlannelBackend string `cli:"flannel-backend"`
FlannelIPv6Masq bool `cli:"flannel-ipv6-masq"`

67
pkg/daemons/control/deps/deps.go
View File

@@ -3,10 +3,10 @@ package deps
import (
"bytes"
"crypto"
cryptorand "crypto/rand"
"crypto/rand"
"crypto/sha256"
"crypto/x509"
b64 "encoding/base64"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
@@ -23,6 +23,7 @@ import (
"github.com/k3s-io/k3s/pkg/cloudprovider"
"github.com/k3s-io/k3s/pkg/daemons/config"
"github.com/k3s-io/k3s/pkg/passwd"
"github.com/k3s-io/k3s/pkg/secretsencrypt"
"github.com/k3s-io/k3s/pkg/util"
"github.com/k3s-io/k3s/pkg/version"
certutil "github.com/rancher/dynamiclistener/cert"
@@ -37,7 +38,6 @@ import (
const (
ipsecTokenSize = 48
aescbcKeySize = 32
RequestHeaderCN = "system:auth-proxy"
)
@@ -725,6 +725,15 @@ func genEncryptionConfigAndState(controlConfig *config.Control) error {
if !controlConfig.EncryptSecrets {
return nil
}
var keyName string
switch controlConfig.EncryptProvider {
case secretsencrypt.AESCBCProvider:
keyName = "aescbckey"
case secretsencrypt.SecretBoxProvider:
keyName = "secretboxkey"
default:
return fmt.Errorf("unsupported secrets-encryption-key-type %s", controlConfig.EncryptProvider)
}
if s, err := os.Stat(runtime.EncryptionConfig); err == nil && s.Size() > 0 {
// On upgrade from older versions, the encryption hash may not exist, create it
if _, err := os.Stat(runtime.EncryptionHash); errors.Is(err, os.ErrNotExist) {
@@ -739,12 +748,40 @@ func genEncryptionConfigAndState(controlConfig *config.Control) error {
return nil
}
aescbcKey := make([]byte, aescbcKeySize)
_, err := cryptorand.Read(aescbcKey)
if err != nil {
keyByte := make([]byte, secretsencrypt.KeySize)
if _, err := rand.Read(keyByte); err != nil {
return err
}
encodedKey := b64.StdEncoding.EncodeToString(aescbcKey)
newKey := []apiserverconfigv1.Key{
{
Name: keyName,
Secret: base64.StdEncoding.EncodeToString(keyByte),
},
}
var provider []apiserverconfigv1.ProviderConfiguration
if controlConfig.EncryptProvider == secretsencrypt.AESCBCProvider {
provider = []apiserverconfigv1.ProviderConfiguration{
{
AESCBC: &apiserverconfigv1.AESConfiguration{
Keys: newKey,
},
},
{
Identity: &apiserverconfigv1.IdentityConfiguration{},
},
}
} else if controlConfig.EncryptProvider == secretsencrypt.SecretBoxProvider {
provider = []apiserverconfigv1.ProviderConfiguration{
{
Secretbox: &apiserverconfigv1.SecretboxConfiguration{
Keys: newKey,
},
},
{
Identity: &apiserverconfigv1.IdentityConfiguration{},
},
}
}
encConfig := apiserverconfigv1.EncryptionConfiguration{
TypeMeta: metav1.TypeMeta{
@@ -754,21 +791,7 @@ func genEncryptionConfigAndState(controlConfig *config.Control) error {
Resources: []apiserverconfigv1.ResourceConfiguration{
{
Resources: []string{"secrets"},
Providers: []apiserverconfigv1.ProviderConfiguration{
{
AESCBC: &apiserverconfigv1.AESConfiguration{
Keys: []apiserverconfigv1.Key{
{
Name: "aescbckey",
Secret: encodedKey,
},
},
},
},
{
Identity: &apiserverconfigv1.IdentityConfiguration{},
},
},
Providers: provider,
},
},
}

121
pkg/secretsencrypt/config.go
View File

@@ -33,6 +33,9 @@ const (
EncryptionReencryptRequest string = "reencrypt_request"
EncryptionReencryptActive string = "reencrypt_active"
EncryptionReencryptFinished string = "reencrypt_finished"
AESCBCProvider string = "aescbc"
SecretBoxProvider string = "secretbox"
KeySize int = 32
SecretListPageSize int64 = 20
SecretQPS float32 = 200
SecretBurst int = 200
@@ -41,6 +44,14 @@ const (
SecretsUpdateCompleteEvent string = "SecretsUpdateComplete"
)
// We support 3 key/provider types: AESCBC, SecretBox, and Identity. The Identity provider is
// represented just as a boolean, which is used to determine if encryption is enabled/disabled.
type EncryptionKeys struct {
AESCBCKeys []apiserverconfigv1.Key
SBKeys []apiserverconfigv1.Key
Identity bool
}
var EncryptionHashAnnotation = version.Program + ".io/encryption-config-hash"
func GetEncryptionProviders(runtime *config.ControlRuntime) ([]apiserverconfigv1.ProviderConfiguration, error) {
@@ -57,63 +68,90 @@ func GetEncryptionProviders(runtime *config.ControlRuntime) ([]apiserverconfigv1
}
// GetEncryptionKeys returns a list of encryption keys from the current encryption configuration.
// If includeIdentity is true, it will also include a fake key representing the identity provider, which
// is used to determine if encryption is enabled/disabled.
func GetEncryptionKeys(runtime *config.ControlRuntime, includeIdentity bool) ([]apiserverconfigv1.Key, error) {
func GetEncryptionKeys(runtime *config.ControlRuntime) (*EncryptionKeys, error) {
currentKeys := &EncryptionKeys{}
providers, err := GetEncryptionProviders(runtime)
if err != nil {
return nil, err
}
if len(providers) > 2 {
return nil, fmt.Errorf("more than 2 providers (%d) found in secrets encryption", len(providers))
if len(providers) > 3 {
return nil, fmt.Errorf("more than 3 providers (%d) found in secrets encryption", len(providers))
}
var curKeys []apiserverconfigv1.Key
for _, p := range providers {
// Since identity doesn't have keys, we make up a fake key to represent it, so we can
// know that encryption is enabled/disabled in the request.
if p.Identity != nil && includeIdentity {
curKeys = append(curKeys, apiserverconfigv1.Key{
Name: "identity",
Secret: "identity",
})
if p.Identity != nil {
currentKeys.Identity = true
}
if p.AESCBC != nil {
curKeys = append(curKeys, p.AESCBC.Keys...)
currentKeys.AESCBCKeys = append(currentKeys.AESCBCKeys, p.AESCBC.Keys...)
}
if p.AESGCM != nil || p.KMS != nil || p.Secretbox != nil {
return nil, fmt.Errorf("non-standard encryption keys found")
if p.Secretbox != nil {
currentKeys.SBKeys = append(currentKeys.SBKeys, p.Secretbox.Keys...)
}
if p.AESGCM != nil || p.KMS != nil {
return nil, fmt.Errorf("unsupported encryption keys found")
}
}
return curKeys, nil
return currentKeys, nil
}
func WriteEncryptionConfig(runtime *config.ControlRuntime, keys []apiserverconfigv1.Key, enable bool) error {
// WriteEncryptionConfig writes the encryption configuration to the file system.
// The provider arg will be placed first, and is used to encrypt new secrets.
func WriteEncryptionConfig(runtime *config.ControlRuntime, keys *EncryptionKeys, provider string, enable bool) error {
// Placing the identity provider first disables encryption
var providers []apiserverconfigv1.ProviderConfiguration
if enable {
providers = []apiserverconfigv1.ProviderConfiguration{
{
AESCBC: &apiserverconfigv1.AESConfiguration{
Keys: keys,
var primary apiserverconfigv1.ProviderConfiguration
var secondary *apiserverconfigv1.ProviderConfiguration
switch provider {
case AESCBCProvider:
primary = apiserverconfigv1.ProviderConfiguration{
AESCBC: &apiserverconfigv1.AESConfiguration{
Keys: keys.AESCBCKeys,
},
}
if len(keys.SBKeys) > 0 {
secondary = &apiserverconfigv1.ProviderConfiguration{
Secretbox: &apiserverconfigv1.SecretboxConfiguration{
Keys: keys.SBKeys,
},
}
}
case SecretBoxProvider:
primary = apiserverconfigv1.ProviderConfiguration{
Secretbox: &apiserverconfigv1.SecretboxConfiguration{
Keys: keys.SBKeys,
},
{
Identity: &apiserverconfigv1.IdentityConfiguration{},
},
}
if len(keys.AESCBCKeys) > 0 {
secondary = &apiserverconfigv1.ProviderConfiguration{
AESCBC: &apiserverconfigv1.AESConfiguration{
Keys: keys.AESCBCKeys,
},
}
}
}
identity := apiserverconfigv1.ProviderConfiguration{
Identity: &apiserverconfigv1.IdentityConfiguration{},
}
// Placing the identity provider first disables encryption
if enable && secondary != nil {
providers = []apiserverconfigv1.ProviderConfiguration{
primary,
*secondary,
identity,
}
} else if enable {
providers = []apiserverconfigv1.ProviderConfiguration{
primary,
identity,
}
} else {
providers = []apiserverconfigv1.ProviderConfiguration{
{
Identity: &apiserverconfigv1.IdentityConfiguration{},
},
{
AESCBC: &apiserverconfigv1.AESConfiguration{
Keys: keys,
},
},
identity,
primary,
}
}
@@ -149,15 +187,24 @@ func GenEncryptionConfigHash(runtime *config.ControlRuntime) (string, error) {
// any identity providers plus a new key based on the input arguments.
func GenReencryptHash(runtime *config.ControlRuntime, keyName string) (string, error) {
keys, err := GetEncryptionKeys(runtime, true)
// To retain compatibility with the older encryption hash format,
// we contruct the hash as: aescbc + secretbox + identity + newkey
currentKeys, err := GetEncryptionKeys(runtime)
if err != nil {
return "", err
}
newKey := apiserverconfigv1.Key{
keys := currentKeys.AESCBCKeys
keys = append(keys, currentKeys.SBKeys...)
if currentKeys.Identity {
keys = append(keys, apiserverconfigv1.Key{
Name: "identity",
Secret: "identity",
})
}
keys = append(keys, apiserverconfigv1.Key{
Name: keyName,
Secret: "12345",
}
keys = append(keys, newKey)
})
b, err := json.Marshal(keys)
if err != nil {
return "", err

169
pkg/server/handlers/secrets-encrypt.go
View File

@@ -32,8 +32,6 @@ import (
"k8s.io/utils/ptr"
)
const aescbcKeySize = 32
type EncryptionState struct {
Stage string `json:"stage"`
ActiveKey string `json:"activekey"`
@@ -89,9 +87,9 @@ func encryptionStatus(control *config.Control) (EncryptionState, error) {
} else if err != nil {
return state, err
}
if providers[1].Identity != nil && providers[0].AESCBC != nil {
if providers[len(providers)-1].Identity != nil && (providers[0].AESCBC != nil || providers[0].Secretbox != nil) {
state.Enable = ptr.To(true)
} else if providers[0].Identity != nil && providers[1].AESCBC != nil || !control.EncryptSecrets {
} else if !control.EncryptSecrets || providers[0].Identity != nil && (providers[1].AESCBC != nil || providers[1].Secretbox != nil) {
state.Enable = ptr.To(false)
}
@@ -110,11 +108,23 @@ func encryptionStatus(control *config.Control) (EncryptionState, error) {
for _, p := range providers {
if p.AESCBC != nil {
for _, aesKey := range p.AESCBC.Keys {
typName := "AES-CBC " + aesKey.Name
if active {
active = false
state.ActiveKey = aesKey.Name
state.ActiveKey = typName
} else {
state.InactiveKeys = append(state.InactiveKeys, aesKey.Name)
state.InactiveKeys = append(state.InactiveKeys, typName)
}
}
}
if p.Secretbox != nil {
for _, sbKey := range p.Secretbox.Keys {
typName := "XSalsa20-POLY1305 " + sbKey.Name
if active {
active = false
state.ActiveKey = typName
} else {
state.InactiveKeys = append(state.InactiveKeys, typName)
}
}
}
@@ -131,24 +141,37 @@ func encryptionEnable(ctx context.Context, control *config.Control, enable bool)
if err != nil {
return err
}
if len(providers) > 2 {
return fmt.Errorf("more than 2 providers (%d) found in secrets encryption", len(providers))
if len(providers) > 3 {
return fmt.Errorf("more than 3 providers (%d) found in secrets encryption", len(providers))
}
curKeys, err := secretsencrypt.GetEncryptionKeys(control.Runtime, false)
curKeys, err := secretsencrypt.GetEncryptionKeys(control.Runtime)
if err != nil {
return err
}
if providers[1].Identity != nil && providers[0].AESCBC != nil && !enable {
if providers[len(providers)-1].Identity != nil && (providers[0].AESCBC != nil || providers[0].Secretbox != nil) && !enable {
logrus.Infoln("Disabling secrets encryption")
if err := secretsencrypt.WriteEncryptionConfig(control.Runtime, curKeys, enable); err != nil {
if err := secretsencrypt.WriteEncryptionConfig(control.Runtime, curKeys, control.EncryptProvider, enable); err != nil {
return err
}
} else if !enable {
logrus.Infoln("Secrets encryption already disabled")
return nil
} else if providers[0].Identity != nil && providers[1].AESCBC != nil && enable {
} else if providers[0].Identity != nil && (providers[1].AESCBC != nil || providers[1].Secretbox != nil) && enable {
foundKey := false
// Check the rest of the providers (generally 2nd and 3rd) for the key type we are trying to enable.
// If we find one, we can proceed.
for _, p := range providers[1:] {
if (control.EncryptProvider == secretsencrypt.AESCBCProvider && p.AESCBC != nil) ||
(control.EncryptProvider == secretsencrypt.SecretBoxProvider && p.Secretbox != nil) {
foundKey = true
}
}
if !foundKey {
return fmt.Errorf("cannot enable secrets encryption with %s key type, no keys found", control.EncryptProvider)
}
logrus.Infoln("Enabling secrets encryption")
if err := secretsencrypt.WriteEncryptionConfig(control.Runtime, curKeys, enable); err != nil {
if err := secretsencrypt.WriteEncryptionConfig(control.Runtime, curKeys, control.EncryptProvider, enable); err != nil {
return err
}
} else if enable {
@@ -208,18 +231,19 @@ func encryptionPrepare(ctx context.Context, control *config.Control, force bool)
if err := verifyEncryptionHashAnnotation(control.Runtime, control.Runtime.Core.Core(), states); err != nil && !force {
return err
}
if control.EncryptProvider == secretsencrypt.SecretBoxProvider {
return fmt.Errorf("prepare does not support secretbox key type, use rotate-keys instead")
}
curKeys, err := secretsencrypt.GetEncryptionKeys(control.Runtime, false)
curKeys, err := secretsencrypt.GetEncryptionKeys(control.Runtime)
if err != nil {
return err
}
if err := AppendNewEncryptionKey(&curKeys); err != nil {
if err := AppendNewEncryptionKey(curKeys, control.EncryptProvider); err != nil {
return err
}
logrus.Infoln("Adding secrets-encryption key: ", curKeys[len(curKeys)-1])
if err := secretsencrypt.WriteEncryptionConfig(control.Runtime, curKeys, true); err != nil {
if err := secretsencrypt.WriteEncryptionConfig(control.Runtime, curKeys, control.EncryptProvider, true); err != nil {
return err
}
nodeName := os.Getenv("NODE_NAME")
@@ -240,19 +264,29 @@ func encryptionRotate(ctx context.Context, control *config.Control, force bool)
if err := verifyEncryptionHashAnnotation(control.Runtime, control.Runtime.Core.Core(), secretsencrypt.EncryptionPrepare); err != nil && !force {
return err
}
if control.EncryptProvider == secretsencrypt.SecretBoxProvider {
return fmt.Errorf("rotate does not support secretbox key type, use rotate-keys instead")
}
curKeys, err := secretsencrypt.GetEncryptionKeys(control.Runtime, false)
curKeys, err := secretsencrypt.GetEncryptionKeys(control.Runtime)
if err != nil {
return err
}
// Right rotate elements
rotatedKeys := append(curKeys[len(curKeys)-1:], curKeys[:len(curKeys)-1]...)
// Right rotate selected keys
switch control.EncryptProvider {
case secretsencrypt.AESCBCProvider:
rotatedKeys := append(curKeys.AESCBCKeys[len(curKeys.AESCBCKeys)-1:], curKeys.AESCBCKeys[:len(curKeys.AESCBCKeys)-1]...)
curKeys.AESCBCKeys = rotatedKeys
case secretsencrypt.SecretBoxProvider:
rotatedKeys := append(curKeys.SBKeys[len(curKeys.SBKeys)-1:], curKeys.SBKeys[:len(curKeys.SBKeys)-1]...)
curKeys.SBKeys = rotatedKeys
}
if err = secretsencrypt.WriteEncryptionConfig(control.Runtime, rotatedKeys, true); err != nil {
if err = secretsencrypt.WriteEncryptionConfig(control.Runtime, curKeys, control.EncryptProvider, true); err != nil {
return err
}
logrus.Infoln("Encryption keys right rotated")
logrus.Infof("Encryption %s keys right rotated\n", control.EncryptProvider)
nodeName := os.Getenv("NODE_NAME")
err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
node, err := control.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{})
@@ -271,6 +305,10 @@ func encryptionReencrypt(ctx context.Context, control *config.Control, force boo
if err := verifyEncryptionHashAnnotation(control.Runtime, control.Runtime.Core.Core(), secretsencrypt.EncryptionRotate); err != nil && !force {
return err
}
if control.EncryptProvider == secretsencrypt.SecretBoxProvider {
return fmt.Errorf("reencrypt does not support secretbox key type, use rotate-keys instead")
}
// Set the reencrypt-active annotation so other nodes know we are in the process of reencrypting.
// As this stage is not persisted, we do not write the annotation to file
nodeName := os.Getenv("NODE_NAME")
@@ -290,25 +328,30 @@ func encryptionReencrypt(ctx context.Context, control *config.Control, force boo
return nil
}
func addAndRotateKeys(control *config.Control) error {
curKeys, err := secretsencrypt.GetEncryptionKeys(control.Runtime, false)
func addAndRotateKeys(control *config.Control, keyType string) error {
curKeys, err := secretsencrypt.GetEncryptionKeys(control.Runtime)
if err != nil {
return err
}
if err := AppendNewEncryptionKey(&curKeys); err != nil {
return err
}
logrus.Infoln("Adding secrets-encryption key: ", curKeys[len(curKeys)-1])
if err := secretsencrypt.WriteEncryptionConfig(control.Runtime, curKeys, true); err != nil {
if err := AppendNewEncryptionKey(curKeys, keyType); err != nil {
return err
}
// Right rotate elements
rotatedKeys := append(curKeys[len(curKeys)-1:], curKeys[:len(curKeys)-1]...)
logrus.Infoln("Rotating secrets-encryption keys")
return secretsencrypt.WriteEncryptionConfig(control.Runtime, rotatedKeys, true)
if err := secretsencrypt.WriteEncryptionConfig(control.Runtime, curKeys, keyType, true); err != nil {
return err
}
// Right rotate keyType keys
if keyType == secretsencrypt.AESCBCProvider {
rotatedKeys := append(curKeys.AESCBCKeys[len(curKeys.AESCBCKeys)-1:], curKeys.AESCBCKeys[:len(curKeys.AESCBCKeys)-1]...)
curKeys.AESCBCKeys = rotatedKeys
} else if keyType == secretsencrypt.SecretBoxProvider {
rotatedKeys := append(curKeys.SBKeys[len(curKeys.SBKeys)-1:], curKeys.SBKeys[:len(curKeys.SBKeys)-1]...)
curKeys.SBKeys = rotatedKeys
}
logrus.Infof("Rotating secrets-encryption %s keys\n", keyType)
return secretsencrypt.WriteEncryptionConfig(control.Runtime, curKeys, keyType, true)
}
// encryptionRotateKeys is both adds and rotates keys, and sets the annotaiton that triggers the
@@ -341,7 +384,7 @@ func encryptionRotateKeys(ctx context.Context, control *config.Control) error {
return err
}
if err := addAndRotateKeys(control); err != nil {
if err := addAndRotateKeys(control, control.EncryptProvider); err != nil {
return err
}
@@ -371,15 +414,33 @@ func reencryptAndRemoveKey(ctx context.Context, control *config.Control, skip bo
return err
}
// Remove last key
curKeys, err := secretsencrypt.GetEncryptionKeys(control.Runtime, false)
// Remove old key. If there is only one of that key type, the cluster just
// migrated between key types. Check for the other key type and remove that.
curKeys, err := secretsencrypt.GetEncryptionKeys(control.Runtime)
if err != nil {
return err
}
logrus.Infoln("Removing key: ", curKeys[len(curKeys)-1])
curKeys = curKeys[:len(curKeys)-1]
if err = secretsencrypt.WriteEncryptionConfig(control.Runtime, curKeys, true); err != nil {
switch control.EncryptProvider {
case secretsencrypt.AESCBCProvider:
if len(curKeys.AESCBCKeys) == 1 && len(curKeys.SBKeys) > 0 {
logrus.Infoln("Removing secretbox key: ", curKeys.SBKeys[len(curKeys.SBKeys)-1])
curKeys.SBKeys = curKeys.SBKeys[:len(curKeys.SBKeys)-1]
} else {
logrus.Infoln("Removing aescbc key: ", curKeys.AESCBCKeys[len(curKeys.AESCBCKeys)-1])
curKeys.AESCBCKeys = curKeys.AESCBCKeys[:len(curKeys.AESCBCKeys)-1]
}
case secretsencrypt.SecretBoxProvider:
if len(curKeys.SBKeys) == 1 && len(curKeys.AESCBCKeys) > 0 {
logrus.Infoln("Removing aescbc key: ", curKeys.AESCBCKeys[len(curKeys.AESCBCKeys)-1])
curKeys.AESCBCKeys = curKeys.AESCBCKeys[:len(curKeys.AESCBCKeys)-1]
} else {
logrus.Infoln("Removing secretbox key: ", curKeys.SBKeys[len(curKeys.SBKeys)-1])
curKeys.SBKeys = curKeys.SBKeys[:len(curKeys.SBKeys)-1]
}
}
if err = secretsencrypt.WriteEncryptionConfig(control.Runtime, curKeys, control.EncryptProvider, true); err != nil {
return err
}
@@ -435,21 +496,33 @@ func updateSecrets(ctx context.Context, control *config.Control, nodeName string
return nil
}
func AppendNewEncryptionKey(keys *[]apiserverconfigv1.Key) error {
aescbcKey := make([]byte, aescbcKeySize)
_, err := rand.Read(aescbcKey)
if err != nil {
func AppendNewEncryptionKey(keys *secretsencrypt.EncryptionKeys, keyType string) error {
var keyPrefix string
switch keyType {
case secretsencrypt.AESCBCProvider:
keyPrefix = "aescbckey-"
case secretsencrypt.SecretBoxProvider:
keyPrefix = "secretboxkey-"
}
keyByte := make([]byte, secretsencrypt.KeySize)
if _, err := rand.Read(keyByte); err != nil {
return err
}
encodedKey := base64.StdEncoding.EncodeToString(aescbcKey)
encodedKey := base64.StdEncoding.EncodeToString(keyByte)
newKey := []apiserverconfigv1.Key{
{
Name: "aescbckey-" + time.Now().Format(time.RFC3339),
Name: keyPrefix + time.Now().Format(time.RFC3339),
Secret: encodedKey,
},
}
*keys = append(*keys, newKey...)
if keyType == secretsencrypt.AESCBCProvider {
keys.AESCBCKeys = append(keys.AESCBCKeys, newKey...)
} else if keyType == secretsencrypt.SecretBoxProvider {
keys.SBKeys = append(keys.SBKeys, newKey...)
}
logrus.Infoln("Adding secrets-encryption key: ", newKey)
return nil
}

46
tests/docker/secretsencryption/secretsencryption_test.go
View File

@@ -169,6 +169,52 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
}
})
})
Context("Switching to Secretbox Provider", func() {
It("Append secretbox provider to config", func() {
cmd := "echo 'secrets-encryption-provider: secretbox' >> /etc/rancher/k3s/config.yaml"
for _, node := range tc.Servers {
Expect(node.RunCmdOnNode(cmd)).Error().NotTo(HaveOccurred())
}
})
It("Restarts K3s servers", func() {
Expect(docker.RestartCluster(tc.Servers)).To(Succeed())
})
It("Rotates the Secrets-Encryption Keys, switching to new key type", func() {
cmd := "k3s secrets-encrypt rotate-keys"
res, err := tc.Servers[0].RunCmdOnNode(cmd)
Expect(err).NotTo(HaveOccurred(), res)
for i, node := range tc.Servers {
Eventually(func(g Gomega) {
cmd := "k3s secrets-encrypt status"
res, err := node.RunCmdOnNode(cmd)
g.Expect(err).NotTo(HaveOccurred(), res)
g.Expect(res).Should(ContainSubstring("Server Encryption Hashes: hash does not match"))
if i == 0 {
g.Expect(res).Should(ContainSubstring("XSalsa20-POLY1305"))
} else {
g.Expect(res).Should(ContainSubstring("AES-CBC"))
}
}, "420s", "10s").Should(Succeed())
}
})
It("Restarts K3s servers", func() {
Expect(docker.RestartCluster(tc.Servers)).To(Succeed())
})
It("Verifies encryption key matches on all nodes", func() {
cmd := "k3s secrets-encrypt status"
for _, node := range tc.Servers {
Eventually(func(g Gomega) {
res, err := node.RunCmdOnNode(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(res).Should(ContainSubstring("XSalsa20-POLY1305"))
g.Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match"))
}, "420s", "2s").Should(Succeed())
}
})
})
})