Adding rocksdb encryption key support

Jenkinsfile.groovy

@@ -27,6 +27,7 @@ pipeline {
     parameters {
       string(name: 'KUBECONFIG', defaultValue: '/home/jenkins/.kube/scw-183a3b', description: 'KUBECONFIG controls which k8s cluster is used', )
       string(name: 'TESTNAMESPACE', defaultValue: 'jenkins', description: 'TESTNAMESPACE sets the kubernetes namespace to ru tests in (this must be short!!)', )
+      string(name: 'ENTERPRISEIMAGE', defaultValue: '', description: 'ENTERPRISEIMAGE sets the docker image used for enterprise tests)', )
     }
     stages {
         stage('Build') {
@@ -44,13 +45,16 @@ pipeline {
             steps {
                 timestamps {
                     lock("${params.TESTNAMESPACE}-${env.GIT_COMMIT}") {
-                        withEnv([
-                        "KUBECONFIG=${params.KUBECONFIG}",
-                        "TESTNAMESPACE=${params.TESTNAMESPACE}-${env.GIT_COMMIT}",
-                        "IMAGETAG=${env.GIT_COMMIT}",
-                        "PUSHIMAGES=1",
-                        ]) {
-                            sh "make run-tests"
+                        withCredentials([string(credentialsId: 'ENTERPRISEIMAGE', variable: 'DEFAULTENTERPRISEIMAGE')]) { 
+                            withEnv([
+                            "KUBECONFIG=${params.KUBECONFIG}",
+                            "TESTNAMESPACE=${params.TESTNAMESPACE}-${env.GIT_COMMIT}",
+                            "IMAGETAG=${env.GIT_COMMIT}",
+                            "PUSHIMAGES=1",
+                            "ENTERPRISEIMAGE=${params.ENTERPRISEIMAGE}",
+                            ]) {
+                                sh "make run-tests"
+                            }
                         }
                     }
                 }

Makefile

@@ -40,6 +40,9 @@ endif
 ifndef TESTIMAGE
 	TESTIMAGE := $(DOCKERNAMESPACE)/arangodb-operator-test$(IMAGESUFFIX)
 endif
+ifndef ENTERPRISEIMAGE
+	ENTERPRISEIMAGE := $(DEFAULTENTERPRISEIMAGE)
+endif
 
 BINNAME := $(PROJECT)
 BIN := $(BINDIR)/$(BINNAME)
@@ -170,7 +173,13 @@ endif
 	kubectl create namespace $(TESTNAMESPACE)
 	$(ROOTDIR)/examples/setup-rbac.sh --namespace=$(TESTNAMESPACE)
 	$(ROOTDIR)/scripts/kube_create_operator.sh $(TESTNAMESPACE) $(OPERATORIMAGE)
-	kubectl --namespace $(TESTNAMESPACE) run arangodb-operator-test -i --rm --quiet --restart=Never --image=$(TESTIMAGE) --env="TEST_NAMESPACE=$(TESTNAMESPACE)" -- -test.v
+	kubectl --namespace $(TESTNAMESPACE) \
+		run arangodb-operator-test -i --rm --quiet --restart=Never \
+		--image=$(TESTIMAGE) \
+		--env="ENTERPRISEIMAGE=$(ENTERPRISEIMAGE)" \
+		--env="TEST_NAMESPACE=$(TESTNAMESPACE)" \
+		-- \
+		-test.v
 	kubectl delete namespace $(TESTNAMESPACE) --ignore-not-found --now
 
 # Release building

docs/user/custom_resource.md

@@ -118,13 +118,19 @@ This requires the Enterprise version.
 
 The encryption key cannot be changed after the cluster has been created.
 
+The secret specified by this setting, must have a data field named 'key' containing
+an encryption key that is exactly 32 bytes long.
+
 ### `spec.auth.jwtSecretName: string`
 
 This setting specifies the name of a kubernetes `Secret` that contains
 the JWT token used for accessing all ArangoDB servers.
 When no name is specified, it defaults to `<deployment-name>-jwt`.
 To disable authentication, set this value to `None`.
 
+If you specify a name of a `Secret`, that secret must have the token
+in a data field named `token`.
+
 If you specify a name of a `Secret` that does not exist, a random token is created
 and stored in a `Secret` with given name.
 

pkg/apis/arangodb/v1alpha/deployment_spec.go

@@ -142,6 +142,12 @@ type RocksDBSpec struct {
 	} `json:"encryption"`
 }
 
+// IsEncrypted returns true when an encryption key secret name is provided,
+// false otherwise.
+func (s RocksDBSpec) IsEncrypted() bool {
+	return s.Encryption.KeySecretName != ""
+}
+
 // Validate the given spec
 func (s RocksDBSpec) Validate() error {
 	if err := k8sutil.ValidateOptionalResourceName(s.Encryption.KeySecretName); err != nil {
@@ -155,6 +161,19 @@ func (s *RocksDBSpec) SetDefaults() {
 	// Nothing needed
 }
 
+// ResetImmutableFields replaces all immutable fields in the given target with values from the source spec.
+// It returns a list of fields that have been reset.
+// Field names are relative to given field prefix.
+func (s RocksDBSpec) ResetImmutableFields(fieldPrefix string, target *RocksDBSpec) []string {
+	var resetFields []string
+	if s.IsEncrypted() != target.IsEncrypted() {
+		// Note: You can change the name, but not from empty to non-empty (or reverse).
+		target.Encryption.KeySecretName = s.Encryption.KeySecretName
+		resetFields = append(resetFields, fieldPrefix+".encryption.keySecretName")
+	}
+	return resetFields
+}
+
 // AuthenticationSpec holds authentication specific configuration settings
 type AuthenticationSpec struct {
 	JWTSecretName string `json:"jwtSecretName,omitempty"`
@@ -519,6 +538,9 @@ func (s DeploymentSpec) ResetImmutableFields(target *DeploymentSpec) []string {
 		target.StorageEngine = s.StorageEngine
 		resetFields = append(resetFields, "storageEngine")
 	}
+	if l := s.RocksDB.ResetImmutableFields("rocksdb", &target.RocksDB); l != nil {
+		resetFields = append(resetFields, l...)
+	}
 	if l := s.Single.ResetImmutableFields(ServerGroupSingle, "single", &target.Single); l != nil {
 		resetFields = append(resetFields, l...)
 	}

pkg/deployment/pod_creator.go

@@ -25,12 +25,14 @@ package deployment
 import (
 	"fmt"
 	"net"
+	"path/filepath"
 	"strconv"
 
 	api "github.com/arangodb/k8s-operator/pkg/apis/arangodb/v1alpha"
 	"github.com/arangodb/k8s-operator/pkg/util/arangod"
 	"github.com/arangodb/k8s-operator/pkg/util/constants"
 	"github.com/arangodb/k8s-operator/pkg/util/k8sutil"
+	"github.com/pkg/errors"
 )
 
 type optionPair struct {
@@ -92,17 +94,11 @@ func (d *Deployment) createArangodArgs(apiObject *api.ArangoDeployment, group ap
 	}*/
 
 	// RocksDB
-	if apiObject.Spec.RocksDB.Encryption.KeySecretName != "" {
-		/*args = append(args,
-			fmt.Sprintf("--rocksdb.encryption-keyfile=%s", apiObject.Spec.StorageEngine),
+	if apiObject.Spec.RocksDB.IsEncrypted() {
+		keyPath := filepath.Join(k8sutil.RocksDBEncryptionVolumeMountDir, "key")
+		options = append(options,
+			optionPair{"--rocksdb.encryption-keyfile", keyPath},
 		)
-		rocksdbSection := &configSection{
-			Name: "rocksdb",
-			Settings: map[string]string{
-				"encryption-keyfile": bsCfg.RocksDBEncryptionKeyFile,
-			},
-		}
-		config = append(config, rocksdbSection)*/
 	}
 
 	options = append(options,
@@ -285,6 +281,7 @@ func (d *Deployment) createReadinessProbe(apiObject *api.ArangoDeployment, group
 // ensurePods creates all Pods listed in member status
 func (d *Deployment) ensurePods(apiObject *api.ArangoDeployment) error {
 	kubecli := d.deps.KubeCli
+	ns := apiObject.GetNamespace()
 
 	if err := apiObject.ForeachServerGroup(func(group api.ServerGroup, spec api.ServerGroupSpec, status *api.MemberStatusList) error {
 		for _, m := range *status {
@@ -304,13 +301,20 @@ func (d *Deployment) ensurePods(apiObject *api.ArangoDeployment) error {
 				if err != nil {
 					return maskAny(err)
 				}
+				rocksdbEncryptionSecretName := ""
+				if apiObject.Spec.RocksDB.IsEncrypted() {
+					rocksdbEncryptionSecretName = apiObject.Spec.RocksDB.Encryption.KeySecretName
+					if err := k8sutil.ValidateEncryptionKeySecret(kubecli, rocksdbEncryptionSecretName, ns); err != nil {
+						return maskAny(errors.Wrapf(err, "RocksDB encryption key secret validation failed"))
+					}
+				}
 				if apiObject.Spec.IsAuthenticated() {
 					env[constants.EnvArangodJWTSecret] = k8sutil.EnvValue{
 						SecretName: apiObject.Spec.Authentication.JWTSecretName,
 						SecretKey:  constants.SecretKeyJWT,
 					}
 				}
-				if err := k8sutil.CreateArangodPod(kubecli, apiObject.Spec.IsDevelopment(), apiObject, role, m.ID, m.PersistentVolumeClaimName, apiObject.Spec.Image, apiObject.Spec.ImagePullPolicy, args, env, livenessProbe, readinessProbe); err != nil {
+				if err := k8sutil.CreateArangodPod(kubecli, apiObject.Spec.IsDevelopment(), apiObject, role, m.ID, m.PersistentVolumeClaimName, apiObject.Spec.Image, apiObject.Spec.ImagePullPolicy, args, env, livenessProbe, readinessProbe, rocksdbEncryptionSecretName); err != nil {
 					return maskAny(err)
 				}
 			} else if group.IsArangosync() {

pkg/util/constants/constants.go

@@ -29,5 +29,6 @@ const (
 	EnvArangodJWTSecret    = "ARANGOD_JWT_SECRET"    // Contains JWT secret for the ArangoDB cluster
 	EnvArangoSyncJWTSecret = "ARANGOSYNC_JWT_SECRET" // Contains JWT secret for the ArangoSync masters
 
-	SecretKeyJWT = "token" // Key inside a Secret used to hold a JW token
+	SecretEncryptionKey = "key"   // Key in a Secret.Data used to store an 32-byte encryption key
+	SecretKeyJWT        = "token" // Key inside a Secret used to hold a JW token
 )

pkg/util/k8sutil/pods.go

@@ -29,8 +29,10 @@ import (
 )
 
 const (
-	arangodVolumeName     = "arangod-data"
-	ArangodVolumeMountDir = "/data"
+	arangodVolumeName               = "arangod-data"
+	rocksdbEncryptionVolumeName     = "rocksdb-encryption"
+	ArangodVolumeMountDir           = "/data"
+	RocksDBEncryptionVolumeMountDir = "/secrets/rocksdb/encryption"
 )
 
 // EnvValue is a helper structure for environment variable sources.
@@ -103,6 +105,16 @@ func arangodVolumeMounts() []v1.VolumeMount {
 	}
 }
 
+// arangodVolumeMounts creates a volume mount structure for arangod.
+func rocksdbEncryptionVolumeMounts() []v1.VolumeMount {
+	return []v1.VolumeMount{
+		{
+			Name:      rocksdbEncryptionVolumeName,
+			MountPath: RocksDBEncryptionVolumeMountDir,
+		},
+	}
+}
+
 // arangodContainer creates a container configured to run `arangod`.
 func arangodContainer(name string, image string, imagePullPolicy v1.PullPolicy, args []string, env map[string]EnvValue, livenessProbe *HTTPProbeConfig, readinessProbe *HTTPProbeConfig) v1.Container {
 	c := v1.Container{
@@ -177,13 +189,19 @@ func newPod(deploymentName, ns, role, id string) v1.Pod {
 // CreateArangodPod creates a Pod that runs `arangod`.
 // If the pod already exists, nil is returned.
 // If another error occurs, that error is returned.
-func CreateArangodPod(kubecli kubernetes.Interface, developmentMode bool, deployment APIObject, role, id, pvcName, image string, imagePullPolicy v1.PullPolicy,
-	args []string, env map[string]EnvValue, livenessProbe *HTTPProbeConfig, readinessProbe *HTTPProbeConfig) error {
+func CreateArangodPod(kubecli kubernetes.Interface, developmentMode bool, deployment APIObject,
+	role, id, pvcName, image string, imagePullPolicy v1.PullPolicy,
+	args []string, env map[string]EnvValue,
+	livenessProbe *HTTPProbeConfig, readinessProbe *HTTPProbeConfig,
+	rocksdbEncryptionSecretName string) error {
 	// Prepare basic pod
 	p := newPod(deployment.GetName(), deployment.GetNamespace(), role, id)
 
 	// Add arangod container
 	c := arangodContainer(p.GetName(), image, imagePullPolicy, args, env, livenessProbe, readinessProbe)
+	if rocksdbEncryptionSecretName != "" {
+		c.VolumeMounts = append(c.VolumeMounts, rocksdbEncryptionVolumeMounts()...)
+	}
 	p.Spec.Containers = append(p.Spec.Containers, c)
 
 	// Add volume
@@ -209,6 +227,19 @@ func CreateArangodPod(kubecli kubernetes.Interface, developmentMode bool, deploy
 		p.Spec.Volumes = append(p.Spec.Volumes, vol)
 	}
 
+	// RocksDB encryption secret mount (if any)
+	if rocksdbEncryptionSecretName != "" {
+		vol := v1.Volume{
+			Name: rocksdbEncryptionVolumeName,
+			VolumeSource: v1.VolumeSource{
+				Secret: &v1.SecretVolumeSource{
+					SecretName: rocksdbEncryptionSecretName,
+				},
+			},
+		}
+		p.Spec.Volumes = append(p.Spec.Volumes, vol)
+	}
+
 	// Add (anti-)affinity
 	p.Spec.Affinity = createAffinity(deployment.GetName(), role, !developmentMode, "")
 

pkg/util/k8sutil/secrets.go

@@ -32,6 +32,42 @@ import (
 	"github.com/arangodb/k8s-operator/pkg/util/constants"
 )
 
+// ValidateEncryptionKeySecret checks that a secret with given name in given namespace
+// exists and it contains a 'key' data field of exactly 32 bytes.
+func ValidateEncryptionKeySecret(kubecli kubernetes.Interface, secretName, namespace string) error {
+	s, err := kubecli.CoreV1().Secrets(namespace).Get(secretName, metav1.GetOptions{})
+	if err != nil {
+		return maskAny(err)
+	}
+	// Check `key` field
+	keyData, found := s.Data[constants.SecretEncryptionKey]
+	if !found {
+		return maskAny(fmt.Errorf("No '%s' found in secret '%s'", constants.SecretEncryptionKey, secretName))
+	}
+	if len(keyData) != 32 {
+		return maskAny(fmt.Errorf("'%s' in secret '%s' is expected to be 32 bytes long, found %d", constants.SecretEncryptionKey, secretName, len(keyData)))
+	}
+	return nil
+}
+
+// CreateEncryptionKeySecret creates a secret used to store a RocksDB encryption key.
+func CreateEncryptionKeySecret(kubecli kubernetes.Interface, secretName, namespace string, key []byte) error {
+	// Create secret
+	secret := &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: secretName,
+		},
+		Data: map[string][]byte{
+			constants.SecretEncryptionKey: key,
+		},
+	}
+	if _, err := kubecli.CoreV1().Secrets(namespace).Create(secret); err != nil {
+		// Failed to create secret
+		return maskAny(err)
+	}
+	return nil
+}
+
 // GetJWTSecret loads the JWT secret from a Secret with given name.
 func GetJWTSecret(kubecli kubernetes.Interface, secretName, namespace string) (string, error) {
 	s, err := kubecli.CoreV1().Secrets(namespace).Get(secretName, metav1.GetOptions{})

tests/rocksdb_encryption_test.go

@@ -0,0 +1,61 @@
+package tests
+
+import (
+	"context"
+	"crypto/rand"
+	"strings"
+	"testing"
+
+	"github.com/dchest/uniuri"
+
+	api "github.com/arangodb/k8s-operator/pkg/apis/arangodb/v1alpha"
+	"github.com/arangodb/k8s-operator/pkg/client"
+	"github.com/arangodb/k8s-operator/pkg/util/k8sutil"
+)
+
+// TestRocksDBEncryptionSingle tests the creating of a single server deployment
+// with RocksDB & Encryption.
+func TestRocksDBEncryptionSingle(t *testing.T) {
+	image := getEnterpriseImageOrSkip(t)
+	c := client.MustNewInCluster()
+	kubecli := mustNewKubeClient(t)
+	ns := getNamespace(t)
+
+	// Prepare deployment config
+	depl := newDeployment("test-rocksdb-enc-sng-" + uniuri.NewLen(4))
+	depl.Spec.Image = image
+	depl.Spec.StorageEngine = api.StorageEngineRocksDB
+	depl.Spec.RocksDB.Encryption.KeySecretName = strings.ToLower(uniuri.New())
+
+	// Create encryption key secret
+	key := make([]byte, 32)
+	rand.Read(key)
+	if err := k8sutil.CreateEncryptionKeySecret(kubecli, depl.Spec.RocksDB.Encryption.KeySecretName, ns, key); err != nil {
+		t.Fatalf("Create encryption key secret failed: %v", err)
+	}
+
+	// Create deployment
+	_, err := c.DatabaseV1alpha().ArangoDeployments(ns).Create(depl)
+	if err != nil {
+		t.Fatalf("Create deployment failed: %v", err)
+	}
+
+	// Wait for deployment to be ready
+	apiObject, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentHasState(api.DeploymentStateRunning))
+	if err != nil {
+		t.Errorf("Deployment not running in time: %v", err)
+	}
+
+	// Create database client
+	ctx := context.Background()
+	client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t)
+
+	// Wait for single server available
+	if err := waitUntilVersionUp(client); err != nil {
+		t.Fatalf("Single server not running returning version in time: %v", err)
+	}
+
+	// Cleanup
+	removeDeployment(c, depl.GetName(), ns)
+	removeSecret(kubecli, depl.Spec.RocksDB.Encryption.KeySecretName, ns)
+}

tests/test_util.go

@@ -52,6 +52,16 @@ var (
 	maskAny = errors.WithStack
 )
 
+// getEnterpriseImageOrSkip returns the docker image used for enterprise
+// tests. If empty, enterprise tests are skipped.
+func getEnterpriseImageOrSkip(t *testing.T) string {
+	image := os.Getenv("ENTERPRISEIMAGE")
+	if image == "" {
+		t.Skip("Skipping test because ENTERPRISEIMAGE is not set")
+	}
+	return image
+}
+
 // mustNewKubeClient creates a kubernetes client
 // failing the test on errors.
 func mustNewKubeClient(t *testing.T) kubernetes.Interface {