add e2e test

Signed-off-by: Bojan Zelic <[email protected]>

Author: BojanZelic

config/e2e/kustomization.yaml

@@ -9,9 +9,15 @@ resources:
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 patches:
-- path: patch_operator.yml
+- path: patch_operator.yaml
   target:
     group: apps
     kind: Deployment
     name: keda-operator
     version: v1
+- path: patch_rbac.yaml
+  target:
+    group: rbac.authorization.k8s.io
+    kind: ClusterRole
+    name: keda-operator
+    version: v1

config/e2e/patch_operator.yml config/e2e/patch_operator.yaml

@@ -0,0 +1,9 @@
+  - op: add
+    path: /rules/-
+    value:
+      apiGroups:
+        - ""
+      resources:
+        - serviceaccounts/token
+      verbs:
+        - create

config/e2e/patch_rbac.yaml

@@ -19,22 +19,19 @@ package resolver
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"os"
 	"strings"
 
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
-
 	"github.com/go-logr/logr"
 	vaultapi "github.com/hashicorp/vault/api"
-	"k8s.io/apimachinery/pkg/types"
-
-	kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1"
+	"github.com/pkg/errors"
 	authenticationv1 "k8s.io/api/authentication/v1"
-
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1"
 )
 
 // HashicorpVaultHandler is specification of Hashi Corp Vault
@@ -141,19 +138,15 @@ func (vh *HashicorpVaultHandler) token(client *vaultapi.Client) (string, error)
 			secret := &corev1.Secret{}
 
 			if err = vh.k8sClient.Get(context.Background(), saName, sa); err != nil {
-				if apierrors.IsNotFound(err) {
-					return token, errors.New(fmt.Sprintf("Failed to retreive service account name: %s namespace: %s", saName.Name, saName.Namespace))
-				}
+				return token, errors.Wrap(err, fmt.Sprintf("Failed to retrieve service account name: %s namespace: %s", saName.Name, saName.Namespace))
 			}
 
 			if len(sa.Secrets) > 0 {
-				//using legacy service account secrets
+				// using legacy service account secrets
 				secretName := types.NamespacedName{Name: sa.Secrets[0].Name, Namespace: vh.namespace}
 
 				if err = vh.k8sClient.Get(context.Background(), secretName, secret); err != nil {
-					if apierrors.IsNotFound(err) {
-						return token, errors.New(fmt.Sprintf("Failed to retreive secret for service account name: %s namespace: %s", secretName.Name, secretName.Namespace))
-					}
+					return token, errors.Wrap(err, fmt.Sprintf("Failed to retrieve secret for service account name: %s namespace: %s", secretName.Name, secretName.Namespace))
 				}
 
 				jwt = secret.Data["token"]
@@ -171,12 +164,11 @@ func (vh *HashicorpVaultHandler) token(client *vaultapi.Client) (string, error)
 				}
 
 				if err := vh.k8sClient.SubResource("token").Create(context.Background(), sa, tokenRequest); err != nil {
-					return token, errors.New(fmt.Sprintf("Failed to create token for service account name: %s namespace: %s", saName.Name, saName.Namespace))
+					return token, errors.Wrap(err, fmt.Sprintf("Failed to create token for service account name: %s namespace: %s", saName.Name, saName.Namespace))
 				}
 
 				jwt = []byte(tokenRequest.Status.Token)
 			}
-
 		} else if len(vh.vault.Credential.ServiceAccount) != 0 {
 			// Get the JWT from POD
 			jwt, err = os.ReadFile(vh.vault.Credential.ServiceAccount)

pkg/scaling/resolver/hashicorpvault_handler.go

@@ -29,14 +29,14 @@ import (
 	vaultapi "github.com/hashicorp/vault/api"
 	"github.com/stretchr/testify/assert"
 	"go.uber.org/mock/gomock"
+	authenticationv1 "k8s.io/api/authentication/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
 	kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1"
 	"github.com/kedacore/keda/v2/pkg/mock/mock_client"
-	authenticationv1 "k8s.io/api/authentication/v1"
-	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
@@ -78,11 +78,6 @@ var (
 		"test":  kedaSecretValue,
 		"array": []string{kedaSecretValue},
 	}
-	kubernetesAuthDataKeda = map[string]interface{}{
-		"auth": map[string]interface{}{
-			"client_token": vaultTestToken,
-		},
-	}
 )
 
 type pkiRequestTestData struct {

pkg/scaling/resolver/hashicorpvault_handler_test.go

@@ -70,6 +70,8 @@ type templateData struct {
 	MonitoredAppName                 string
 	PrometheusServerName             string
 	VaultPkiCommonName               string
+	VaultRole                        string
+	VaultServiceAccountName          string
 }
 
 const (
@@ -128,9 +130,12 @@ metadata:
 spec:
   hashiCorpVault:
     address: http://vault.{{.VaultNamespace}}:8200
-    authentication: token
+    authentication: {{.HashiCorpAuthentication}}
+    role: {{.VaultRole}}
+    mount: kubernetes
     credential:
       token: {{.HashiCorpToken}}
+      serviceAccountName: {{.VaultServiceAccountName}}
     secrets:
     - parameter: connection
       key: connectionString
@@ -413,6 +418,13 @@ spec:
 	pkiPolicyTemplate = `path "pki*" {
   capabilities = [ "create", "read", "update", "delete", "list", "sudo" ]
 }`
+
+	secretReadPolicyTemplate = `path "secret/data/keda" {
+    capabilities = ["read"]
+}
+path "secret/metadata/keda" {
+    capabilities = ["read", "list"]
+}`
 )
 
 func TestPkiSecretsEngine(t *testing.T) {
@@ -432,7 +444,7 @@ func TestPkiSecretsEngine(t *testing.T) {
 			// Create kubernetes resources
 			kc := GetKubernetesClient(t)
 			useKubernetesAuth := test.authentication == "kubernetes"
-			hashiCorpToken, promPkiData := setupHashiCorpVault(t, kc, 2, useKubernetesAuth, true)
+			hashiCorpToken, promPkiData := setupHashiCorpVault(t, kc, 2, useKubernetesAuth, true, false)
 			prometheus.Install(t, kc, prometheusServerName, testNamespace, promPkiData)
 
 			// Create kubernetes resources for testing
@@ -460,16 +472,29 @@ func TestSecretsEngine(t *testing.T) {
 		name               string
 		vaultEngineVersion uint
 		vaultSecretPath    string
+		useKubernetesAuth  bool
+		useDelegatesSAAuth bool
 	}{
 		{
 			name:               "vault kv engine v1",
 			vaultEngineVersion: 1,
 			vaultSecretPath:    "secret/keda",
+			useKubernetesAuth:  false,
+			useDelegatesSAAuth: false,
+		},
+		{
+			name:               "vault kv engine v2",
+			vaultEngineVersion: 2,
+			vaultSecretPath:    "secret/data/keda",
+			useKubernetesAuth:  false,
+			useDelegatesSAAuth: false,
 		},
 		{
 			name:               "vault kv engine v2",
 			vaultEngineVersion: 2,
 			vaultSecretPath:    "secret/data/keda",
+			useKubernetesAuth:  true,
+			useDelegatesSAAuth: true,
 		},
 	}
 
@@ -480,7 +505,7 @@ func TestSecretsEngine(t *testing.T) {
 			data, postgreSQLtemplates := getPostgreSQLTemplateData()
 
 			CreateKubernetesResources(t, kc, testNamespace, data, postgreSQLtemplates)
-			hashiCorpToken, _ := setupHashiCorpVault(t, kc, test.vaultEngineVersion, false, false)
+			hashiCorpToken, _ := setupHashiCorpVault(t, kc, test.vaultEngineVersion, test.useKubernetesAuth, false, test.useDelegatesSAAuth)
 
 			assert.True(t, WaitForStatefulsetReplicaReadyCount(t, kc, postgreSQLStatefulSetName, testNamespace, 1, 60, 3),
 				"replica count should be %d after 3 minutes", 1)
@@ -493,8 +518,19 @@ func TestSecretsEngine(t *testing.T) {
 
 			// Create kubernetes resources for testing
 			data, templates := getTemplateData()
-			data.HashiCorpToken = RemoveANSI(hashiCorpToken)
 			data.VaultSecretPath = test.vaultSecretPath
+			data.VaultRole = "keda"
+			if test.useKubernetesAuth {
+				data.HashiCorpAuthentication = "kubernetes"
+			} else {
+				data.HashiCorpAuthentication = "token"
+				data.HashiCorpToken = RemoveANSI(hashiCorpToken)
+			}
+
+			if test.useDelegatesSAAuth {
+				data.VaultRole = "vault-delegated-sa"
+				data.VaultServiceAccountName = "default"
+			}
 
 			KubectlApplyMultipleWithTemplate(t, data, templates)
 			assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 60, 3),
@@ -548,7 +584,7 @@ func setupHashiCorpVaultPki(t *testing.T, podName string, nameSpace string) *pro
 	return &pkiData
 }
 
-func setupHashiCorpVault(t *testing.T, kc *kubernetes.Clientset, kvVersion uint, useKubernetesAuth, pki bool) (string, *prometheus.VaultPkiData) {
+func setupHashiCorpVault(t *testing.T, kc *kubernetes.Clientset, kvVersion uint, useKubernetesAuth, pki, delegatedAuth bool) (string, *prometheus.VaultPkiData) {
 	CreateNamespace(t, kc, vaultNamespace)
 
 	_, err := ExecuteCommand("helm repo add hashicorp https://helm.releases.hashicorp.com")
@@ -572,7 +608,7 @@ func setupHashiCorpVault(t *testing.T, kc *kubernetes.Clientset, kvVersion uint,
 	// Enable Kubernetes auth
 	if useKubernetesAuth {
 		if pki {
-			remoteFile := "/tmp/policy.hcl"
+			remoteFile := "/tmp/pki_policy.hcl"
 			KubectlCopyToPod(t, pkiPolicyTemplate, remoteFile, podName, vaultNamespace)
 			assert.NoErrorf(t, err, "cannot create policy file in hashicorp vault - %s", err)
 			_, _, err = ExecCommandOnSpecificPod(t, podName, vaultNamespace, fmt.Sprintf("vault policy write pkiPolicy %s", remoteFile))
@@ -584,9 +620,21 @@ func setupHashiCorpVault(t *testing.T, kc *kubernetes.Clientset, kvVersion uint,
 		assert.NoErrorf(t, err, "cannot set kubernetes host in hashicorp vault - %s", err)
 		_, _, err = ExecCommandOnSpecificPod(t, podName, vaultNamespace, "vault write auth/kubernetes/role/keda bound_service_account_names=keda-operator bound_service_account_namespaces=keda policies=pkiPolicy ttl=1h")
 		assert.NoErrorf(t, err, "cannot cerate keda role in hashicorp vault - %s", err)
+		if delegatedAuth {
+			remoteFile := "/tmp/secret_read_policy.hcl"
+			KubectlCopyToPod(t, secretReadPolicyTemplate, remoteFile, podName, vaultNamespace)
+			assert.NoErrorf(t, err, "cannot create policy file in hashicorp vault - %s", err)
+			_, _, err = ExecCommandOnSpecificPod(t, podName, vaultNamespace, fmt.Sprintf("vault policy write secretReadPolicy %s", remoteFile))
+			assert.NoErrorf(t, err, "cannot create policy in hashicorp vault - %s", err)
+
+			_, _, err = ExecCommandOnSpecificPod(t, podName, vaultNamespace, fmt.Sprintf("vault write auth/kubernetes/role/vault-delegated-sa bound_service_account_names=default bound_service_account_namespaces=%s policies=secretReadPolicy ttl=1h", testNamespace))
+			assert.NoErrorf(t, err, "cannot cerate keda role in hashicorp vault - %s", err)
+		}
 	}
+
 	// Create kv secret
 	if !pki {
+		t.Logf("added secret")
 		_, _, err = ExecCommandOnSpecificPod(t, podName, vaultNamespace, fmt.Sprintf("vault kv put secret/keda connectionString=%s", postgreSQLConnectionString))
 		assert.NoErrorf(t, err, "cannot put connection string in hashicorp vault - %s", err)
 	}