From 550ace82ecfab7a322b881c6dc08eccd776edfcb Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Mon, 19 Mar 2018 11:09:20 +0100 Subject: [PATCH 1/4] Splitting operator in two parts --- Makefile | 17 ++- README.md | 4 +- docs/user/usage.md | 12 +- main.go | 82 ++++-------- ...erator.yaml => arango-deployment-dev.yaml} | 31 ++--- manifests/arango-storage-dev.yaml | 86 ++++++++++++ .../templates/deployment/deployment.yaml | 28 ++++ .../templates/{ => deployment}/rbac.yaml | 20 +-- .../templates/{ => storage}/deployment.yaml | 12 +- manifests/templates/storage/rbac.yaml | 56 ++++++++ pkg/operator/crd.go | 39 +++--- pkg/operator/operator.go | 70 ++++++---- pkg/operator/operator_deployment.go | 7 +- pkg/operator/operator_leader.go | 60 +++++++++ pkg/operator/operator_local_storage.go | 7 +- tools/manifests/manifest_builder.go | 126 +++++++++++------- 16 files changed, 452 insertions(+), 205 deletions(-) rename manifests/{arango-operator.yaml => arango-deployment-dev.yaml} (73%) create mode 100644 manifests/arango-storage-dev.yaml create mode 100644 manifests/templates/deployment/deployment.yaml rename manifests/templates/{ => deployment}/rbac.yaml (74%) rename manifests/templates/{ => storage}/deployment.yaml (67%) create mode 100644 manifests/templates/storage/rbac.yaml create mode 100644 pkg/operator/operator_leader.go diff --git a/Makefile b/Makefile index 3088e7c0e..b46f3bc77 100644 --- a/Makefile +++ b/Makefile @@ -40,9 +40,11 @@ else IMAGESUFFIX := :dev endif -ifndef MANIFESTPATH - MANIFESTPATH := manifests/arango-operator-dev.yaml +ifndef MANIFESTSUFFIX + MANIFESTSUFFIX := -dev endif +MANIFESTPATHDEPLOYMENT := manifests/arango-deployment$(MANIFESTSUFFIX).yaml +MANIFESTPATHSTORAGE := manifests/arango-storage$(MANIFESTSUFFIX).yaml ifndef DEPLOYMENTNAMESPACE DEPLOYMENTNAMESPACE := default endif @@ -189,7 +191,7 @@ endif .PHONY: manifests manifests: $(GOBUILDDIR) GOPATH=$(GOBUILDDIR) go run $(ROOTDIR)/tools/manifests/manifest_builder.go \ - --output=$(MANIFESTPATH) \ + --output-suffix=$(MANIFESTSUFFIX) \ --image=$(OPERATORIMAGE) \ --image-sha256=$(IMAGESHA256) \ --namespace=$(DEPLOYMENTNAMESPACE) @@ -240,7 +242,8 @@ ifneq ($(DEPLOYMENTNAMESPACE), default) $(ROOTDIR)/scripts/kube_delete_namespace.sh $(DEPLOYMENTNAMESPACE) kubectl create namespace $(DEPLOYMENTNAMESPACE) endif - kubectl apply -f $(MANIFESTPATH) + kubectl apply -f $(MANIFESTPATHSTORAGE) + kubectl apply -f $(MANIFESTPATHDEPLOYMENT) $(ROOTDIR)/scripts/kube_create_storage.sh $(DEPLOYMENTNAMESPACE) kubectl --namespace $(DEPLOYMENTNAMESPACE) \ run arangodb-operator-test -i --rm --quiet --restart=Never \ @@ -308,9 +311,11 @@ minikube-start: .PHONY: delete-operator delete-operator: - kubectl delete -f $(MANIFESTPATH) --ignore-not-found + kubectl delete -f $(MANIFESTPATHDEPLOYMENT) --ignore-not-found + kubectl delete -f $(MANIFESTPATHSTORAGE) --ignore-not-found .PHONY: redeploy-operator redeploy-operator: delete-operator manifests - kubectl apply -f $(MANIFESTPATH) + kubectl apply -f $(MANIFESTPATHSTORAGE) + kubectl apply -f $(MANIFESTPATHDEPLOYMENT) kubectl get pods diff --git a/README.md b/README.md index 49a40b351..f5ff5cf91 100644 --- a/README.md +++ b/README.md @@ -11,5 +11,7 @@ State: In heavy development. DO NOT USE FOR ANY PRODUCTION LIKE PURPOSE! THINGS ```bash DOCKERNAMESPACE= make -kubectl apply -f manifests/arango-operator-dev.yaml +kubectl apply -f manifests/arango-deployment-dev.yaml +# To use `ArangoLocalStorage`, also run +kubectl apply -f manifests/arango-storage-dev.yaml ``` diff --git a/docs/user/usage.md b/docs/user/usage.md index d33d76667..469efb0d1 100644 --- a/docs/user/usage.md +++ b/docs/user/usage.md @@ -6,7 +6,13 @@ The ArangoDB operator needs to be installed in your Kubernetes cluster first. To do so, clone this repository and run: ```bash -kubectl apply -f manifests/arango-operator.yaml +kubectl apply -f manifests/arango-deployment.yaml +``` + +To use `ArangoLocalStorage`, also run: + +```bash +kubectl apply -f manifests/arango-storage.yaml ``` ## Cluster creation @@ -37,5 +43,7 @@ To remove the entire ArangoDB operator, remove all clusters first and then remove the operator by running: ```bash -kubectl delete -f manifests/arango-operator.yaml +kubectl delete -f manifests/arango-deployment.yaml +# If `ArangoLocalStorage` is installed +kubectl delete -f manifests/arango-storage.yaml ``` diff --git a/main.go b/main.go index 0ccf7752f..5f1e9019a 100644 --- a/main.go +++ b/main.go @@ -41,8 +41,6 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" v1core "k8s.io/client-go/kubernetes/typed/core/v1" - "k8s.io/client-go/tools/leaderelection" - "k8s.io/client-go/tools/leaderelection/resourcelock" "k8s.io/client-go/tools/record" "github.com/arangodb/kube-arangodb/pkg/client" @@ -77,7 +75,11 @@ var ( host string port int } - createCRD bool + operatorOptions struct { + enableDeployment bool // Run deployment operator + enableStorage bool // Run deployment operator + createCRD bool + } ) func init() { @@ -85,7 +87,9 @@ func init() { f.StringVar(&server.host, "server.host", defaultServerHost, "Host to listen on") f.IntVar(&server.port, "server.port", defaultServerPort, "Port to listen on") f.StringVar(&logLevel, "log.level", defaultLogLevel, "Set initial log level") - f.BoolVar(&createCRD, "operator.create-crd", true, "Disable to avoid create the custom resource definition") + f.BoolVar(&operatorOptions.enableDeployment, "operator.deployment", false, "Enable to run the ArangoDeployment operator") + f.BoolVar(&operatorOptions.enableStorage, "operator.storage", false, "Enable to run the ArangoLocalStorage operator") + f.BoolVar(&operatorOptions.createCRD, "operator.create-crd", true, "Disable to avoid create the custom resource definition") } func main() { @@ -107,6 +111,11 @@ func cmdMainRun(cmd *cobra.Command, args []string) { cliLog.Fatal().Err(err).Msg("Failed to initialize log service") } + // Check operating mode + if !operatorOptions.enableDeployment && !operatorOptions.enableStorage { + cliLog.Fatal().Err(err).Msg("Turn on --operator.deployment or --operator.storage or both") + } + // Log version cliLog.Info().Msgf("Starting arangodb-operator, version %s build %s", projectVersion, projectBuild) @@ -126,48 +135,12 @@ func cmdMainRun(cmd *cobra.Command, args []string) { cliLog.Fatal().Err(err).Msg("Failed to get hostname") } - // Create k8s client - kubecli, err := k8sutil.NewKubeClient() - if err != nil { - cliLog.Fatal().Err(err).Msg("Failed to create kubernetes client") - } - //http.HandleFunc(probe.HTTPReadyzEndpoint, probe.ReadyzHandler) http.Handle("/metrics", prometheus.Handler()) listenAddr := net.JoinHostPort(server.host, strconv.Itoa(server.port)) go http.ListenAndServe(listenAddr, nil) - rl, err := resourcelock.New(resourcelock.EndpointsResourceLock, - namespace, - "arangodb-operator", - kubecli.CoreV1(), - resourcelock.ResourceLockConfig{ - Identity: id, - EventRecorder: createRecorder(cliLog, kubecli, name, namespace), - }) - if err != nil { - cliLog.Fatal().Err(err).Msg("Failed to create resource lock") - } - - leaderelection.RunOrDie(leaderelection.LeaderElectionConfig{ - Lock: rl, - LeaseDuration: 15 * time.Second, - RenewDeadline: 10 * time.Second, - RetryPeriod: 2 * time.Second, - Callbacks: leaderelection.LeaderCallbacks{ - OnStartedLeading: func(stop <-chan struct{}) { - run(stop, namespace, name) - }, - OnStoppedLeading: func() { - cliLog.Fatal().Msg("Leader election lost") - }, - }, - }) -} - -// run the operator -func run(stop <-chan struct{}, namespace, name string) { - cfg, deps, err := newOperatorConfigAndDeps(namespace, name) + cfg, deps, err := newOperatorConfigAndDeps(id+"-"+name, namespace, name) if err != nil { cliLog.Fatal().Err(err).Msg("Failed to create operator config & deps") } @@ -178,13 +151,11 @@ func run(stop <-chan struct{}, namespace, name string) { if err != nil { cliLog.Fatal().Err(err).Msg("Failed to create operator") } - if err := o.Start(); err != nil { - cliLog.Fatal().Err(err).Msg("Failed to start operator") - } + o.Run() } // newOperatorConfigAndDeps creates operator config & dependencies. -func newOperatorConfigAndDeps(namespace, name string) (operator.Config, operator.Dependencies, error) { +func newOperatorConfigAndDeps(id, namespace, name string) (operator.Config, operator.Dependencies, error) { kubecli, err := k8sutil.NewKubeClient() if err != nil { return operator.Config{}, operator.Dependencies{}, maskAny(err) @@ -203,18 +174,23 @@ func newOperatorConfigAndDeps(namespace, name string) (operator.Config, operator if err != nil { return operator.Config{}, operator.Dependencies{}, maskAny(fmt.Errorf("Failed to created versioned client: %s", err)) } + eventRecorder := createRecorder(cliLog, kubecli, name, namespace) cfg := operator.Config{ - Namespace: namespace, - PodName: name, - ServiceAccount: serviceAccount, - CreateCRD: createCRD, + ID: id, + Namespace: namespace, + PodName: name, + ServiceAccount: serviceAccount, + EnableDeployment: operatorOptions.enableDeployment, + EnableStorage: operatorOptions.enableStorage, + CreateCRD: operatorOptions.createCRD, } deps := operator.Dependencies{ - Log: logService.MustGetLogger("operator"), - KubeCli: kubecli, - KubeExtCli: kubeExtCli, - CRCli: crCli, + Log: logService.MustGetLogger("operator"), + KubeCli: kubecli, + KubeExtCli: kubeExtCli, + CRCli: crCli, + EventRecorder: eventRecorder, } return cfg, deps, nil diff --git a/manifests/arango-operator.yaml b/manifests/arango-deployment-dev.yaml similarity index 73% rename from manifests/arango-operator.yaml rename to manifests/arango-deployment-dev.yaml index a6326f912..e1f4c76ce 100644 --- a/manifests/arango-operator.yaml +++ b/manifests/arango-deployment-dev.yaml @@ -1,8 +1,8 @@ -## rbac.yaml +## deployment/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: - name: arango-operator + name: arango-deployment-operator rules: - apiGroups: - database.arangodb.com @@ -10,12 +10,6 @@ rules: - arangodeployments verbs: - "*" -- apiGroups: - - storage.arangodb.com - resources: - - arangolocalstorages - verbs: - - "*" - apiGroups: - apiextensions.k8s.io resources: @@ -28,7 +22,6 @@ rules: - pods - services - endpoints - - persistentvolumes - persistentvolumeclaims - events - secrets @@ -38,7 +31,6 @@ rules: - apps resources: - deployments - - daemonsets verbs: - "*" - apiGroups: @@ -46,18 +38,19 @@ rules: resources: - storageclasses verbs: - - "*" + - get + - list --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: - name: arango-operator + name: arango-deployment-operator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: arango-operator + name: arango-deployment-operator subjects: - kind: ServiceAccount name: default @@ -65,24 +58,26 @@ subjects: --- -## deployment.yaml +## deployment/deployment.yaml apiVersion: extensions/v1beta1 kind: Deployment metadata: - name: arango-operator + name: arango-deployment-operator namespace: default spec: replicas: 1 template: metadata: labels: - name: arango-operator + name: arango-deployment-operator spec: containers: - - name: arangodb-operator + - name: operator imagePullPolicy: IfNotPresent - image: arangodb/arangodb-operator@sha256:90663ccc4cc71562ec06b7e2ad189762aaa38e922e0c70b3cc12f2afa51bf50c + image: ewoutp/arangodb-operator@sha256:51a122f8b36cb26f181dd1ffdddae6cb3fb5a4def4709c3ea123744a557cbe08 + args: + - --operator.deployment env: - name: MY_POD_NAMESPACE valueFrom: diff --git a/manifests/arango-storage-dev.yaml b/manifests/arango-storage-dev.yaml new file mode 100644 index 000000000..eaf7d3487 --- /dev/null +++ b/manifests/arango-storage-dev.yaml @@ -0,0 +1,86 @@ +## storage/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: arango-deployment-operator +rules: +- apiGroups: + - storage.arangodb.com + resources: + - arangolocalstorages + verbs: + - "*" +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - "*" +- apiGroups: + - "" + resources: + - persistentvolumes + - persistentvolumeclaims + - events + verbs: + - "*" +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - "*" +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - "*" + +--- + +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: arango-deployment-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: arango-deployment-operator +subjects: +- kind: ServiceAccount + name: default + namespace: default + +--- + +## storage/deployment.yaml + +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: arango-storage-operator + namespace: default +spec: + replicas: 1 + template: + metadata: + labels: + name: arango-storage-operator + spec: + containers: + - name: operator + imagePullPolicy: IfNotPresent + image: ewoutp/arangodb-operator@sha256:51a122f8b36cb26f181dd1ffdddae6cb3fb5a4def4709c3ea123744a557cbe08 + args: + - --operator.storage + env: + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + diff --git a/manifests/templates/deployment/deployment.yaml b/manifests/templates/deployment/deployment.yaml new file mode 100644 index 000000000..0d6f96e3f --- /dev/null +++ b/manifests/templates/deployment/deployment.yaml @@ -0,0 +1,28 @@ + +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: {{ .Deployment.OperatorName }} + namespace: {{ .Deployment.Namespace }} +spec: + replicas: 1 + template: + metadata: + labels: + name: {{ .Deployment.OperatorName }} + spec: + containers: + - name: operator + imagePullPolicy: {{ .ImagePullPolicy }} + image: {{ .Image }} + args: + - --operator.deployment + env: + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name diff --git a/manifests/templates/rbac.yaml b/manifests/templates/deployment/rbac.yaml similarity index 74% rename from manifests/templates/rbac.yaml rename to manifests/templates/deployment/rbac.yaml index 22d3a22e6..9d9590bfd 100644 --- a/manifests/templates/rbac.yaml +++ b/manifests/templates/deployment/rbac.yaml @@ -1,9 +1,8 @@ - {{- if .RBAC -}} apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: - name: {{ .ClusterRoleName }} + name: {{ .Deployment.ClusterRoleName }} rules: - apiGroups: - database.arangodb.com @@ -11,12 +10,6 @@ rules: - arangodeployments verbs: - "*" -- apiGroups: - - storage.arangodb.com - resources: - - arangolocalstorages - verbs: - - "*" - apiGroups: - apiextensions.k8s.io resources: @@ -29,7 +22,6 @@ rules: - pods - services - endpoints - - persistentvolumes - persistentvolumeclaims - events - secrets @@ -39,7 +31,6 @@ rules: - apps resources: - deployments - - daemonsets verbs: - "*" - apiGroups: @@ -47,21 +38,22 @@ rules: resources: - storageclasses verbs: - - "*" + - get + - list --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: - name: {{ .ClusterRoleBindingName }} + name: {{ .Deployment.ClusterRoleBindingName }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: {{ .ClusterRoleName }} + name: {{ .Deployment.ClusterRoleName }} subjects: - kind: ServiceAccount name: default - namespace: {{ .Namespace }} + namespace: {{ .Deployment.Namespace }} {{- end -}} \ No newline at end of file diff --git a/manifests/templates/deployment.yaml b/manifests/templates/storage/deployment.yaml similarity index 67% rename from manifests/templates/deployment.yaml rename to manifests/templates/storage/deployment.yaml index 89bc24224..9d94b9ea7 100644 --- a/manifests/templates/deployment.yaml +++ b/manifests/templates/storage/deployment.yaml @@ -2,19 +2,21 @@ apiVersion: extensions/v1beta1 kind: Deployment metadata: - name: {{ .OperatorName }} - namespace: {{ .Namespace }} + name: {{ .Storage.OperatorName }} + namespace: {{ .Storage.Namespace }} spec: replicas: 1 template: metadata: labels: - name: {{ .OperatorName }} + name: {{ .Storage.OperatorName }} spec: containers: - - name: arangodb-operator + - name: operator imagePullPolicy: {{ .ImagePullPolicy }} - image: {{ .OperatorImage }} + image: {{ .Image }} + args: + - --operator.storage env: - name: MY_POD_NAMESPACE valueFrom: diff --git a/manifests/templates/storage/rbac.yaml b/manifests/templates/storage/rbac.yaml new file mode 100644 index 000000000..aea2a509e --- /dev/null +++ b/manifests/templates/storage/rbac.yaml @@ -0,0 +1,56 @@ + +{{- if .RBAC -}} +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: {{ .Deployment.ClusterRoleName }} +rules: +- apiGroups: + - storage.arangodb.com + resources: + - arangolocalstorages + verbs: + - "*" +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - "*" +- apiGroups: + - "" + resources: + - persistentvolumes + - persistentvolumeclaims + - events + verbs: + - "*" +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - "*" +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - "*" + +--- + +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: {{ .Deployment.ClusterRoleBindingName }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Deployment.ClusterRoleName }} +subjects: +- kind: ServiceAccount + name: default + namespace: {{ .Deployment.Namespace }} + +{{- end -}} \ No newline at end of file diff --git a/pkg/operator/crd.go b/pkg/operator/crd.go index 9c8dcb3dc..965ff0132 100644 --- a/pkg/operator/crd.go +++ b/pkg/operator/crd.go @@ -34,9 +34,9 @@ import ( // initResourceIfNeeded initializes the custom resource definition when // instructed to do so by the config. -func (o *Operator) initResourceIfNeeded() error { +func (o *Operator) initResourceIfNeeded(enableDeployment, enableStorage bool) error { if o.Config.CreateCRD { - if err := o.initCRD(); err != nil { + if err := o.initCRD(enableDeployment, enableStorage); err != nil { return maskAny(fmt.Errorf("Failed to initialize Custom Resource Definition: %v", err)) } } @@ -44,25 +44,30 @@ func (o *Operator) initResourceIfNeeded() error { } // initCRD creates the CustomResourceDefinition and waits for it to be ready. -func (o *Operator) initCRD() error { +func (o *Operator) initCRD(enableDeployment, enableStorage bool) error { log := o.Dependencies.Log - log.Debug().Msg("Creating ArangoDeployment CRD") - if err := crd.CreateCRD(o.KubeExtCli, deplapi.SchemeGroupVersion, deplapi.ArangoDeploymentCRDName, deplapi.ArangoDeploymentResourceKind, deplapi.ArangoDeploymentResourcePlural, deplapi.ArangoDeploymentShortNames...); err != nil { - return maskAny(errors.Wrapf(err, "failed to create CRD: %v", err)) - } - log.Debug().Msg("Waiting for ArangoDeployment CRD to be ready") - if err := crd.WaitCRDReady(o.KubeExtCli, deplapi.ArangoDeploymentCRDName); err != nil { - return maskAny(err) + if enableDeployment { + log.Debug().Msg("Creating ArangoDeployment CRD") + if err := crd.CreateCRD(o.KubeExtCli, deplapi.SchemeGroupVersion, deplapi.ArangoDeploymentCRDName, deplapi.ArangoDeploymentResourceKind, deplapi.ArangoDeploymentResourcePlural, deplapi.ArangoDeploymentShortNames...); err != nil { + return maskAny(errors.Wrapf(err, "failed to create CRD: %v", err)) + } + log.Debug().Msg("Waiting for ArangoDeployment CRD to be ready") + if err := crd.WaitCRDReady(o.KubeExtCli, deplapi.ArangoDeploymentCRDName); err != nil { + return maskAny(err) + } } - log.Debug().Msg("Creating ArangoLocalStorage CRD") - if err := crd.CreateCRD(o.KubeExtCli, lsapi.SchemeGroupVersion, lsapi.ArangoLocalStorageCRDName, lsapi.ArangoLocalStorageResourceKind, lsapi.ArangoLocalStorageResourcePlural, lsapi.ArangoLocalStorageShortNames...); err != nil { - return maskAny(errors.Wrapf(err, "failed to create CRD: %v", err)) - } - log.Debug().Msg("Waiting for ArangoLocalStorage CRD to be ready") - if err := crd.WaitCRDReady(o.KubeExtCli, lsapi.ArangoLocalStorageCRDName); err != nil { - return maskAny(err) + if enableStorage { + log.Debug().Msg("Creating ArangoLocalStorage CRD") + if err := crd.CreateCRD(o.KubeExtCli, lsapi.SchemeGroupVersion, lsapi.ArangoLocalStorageCRDName, lsapi.ArangoLocalStorageResourceKind, lsapi.ArangoLocalStorageResourcePlural, lsapi.ArangoLocalStorageShortNames...); err != nil { + return maskAny(errors.Wrapf(err, "failed to create CRD: %v", err)) + } + log.Debug().Msg("Waiting for ArangoLocalStorage CRD to be ready") + if err := crd.WaitCRDReady(o.KubeExtCli, lsapi.ArangoLocalStorageCRDName); err != nil { + return maskAny(err) + } } + return nil } diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index d767f7cb8..0ab8d78cc 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -27,9 +27,11 @@ import ( "time" "github.com/rs/zerolog" + "github.com/rs/zerolog/log" apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" kwatch "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/record" deplapi "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" lsapi "github.com/arangodb/kube-arangodb/pkg/apis/storage/v1alpha" @@ -57,17 +59,21 @@ type Operator struct { } type Config struct { - Namespace string - PodName string - ServiceAccount string - CreateCRD bool + ID string + Namespace string + PodName string + ServiceAccount string + EnableDeployment bool + EnableStorage bool + CreateCRD bool } type Dependencies struct { - Log zerolog.Logger - KubeCli kubernetes.Interface - KubeExtCli apiextensionsclient.Interface - CRCli versioned.Interface + Log zerolog.Logger + KubeCli kubernetes.Interface + KubeExtCli apiextensionsclient.Interface + CRCli versioned.Interface + EventRecorder record.EventRecorder } // NewOperator instantiates a new operator from given config & dependencies. @@ -81,12 +87,22 @@ func NewOperator(config Config, deps Dependencies) (*Operator, error) { return o, nil } -// Start the operator -func (o *Operator) Start() error { - log := o.Dependencies.Log +// Run the operator +func (o *Operator) Run() { + if o.Config.EnableDeployment { + go o.runLeaderElection("arango-deployment-operator", o.onStartDeployment) + } + if o.Config.EnableStorage { + go o.runLeaderElection("arango-storage-operator", o.onStartStorage) + } + // Wait until process terminates + <-context.TODO().Done() +} +// onStartDeployment starts the deployment operator and run till given channel is closed. +func (o *Operator) onStartDeployment(stop <-chan struct{}) { for { - if err := o.initResourceIfNeeded(); err == nil { + if err := o.initResourceIfNeeded(true, false); err == nil { break } else { log.Error().Err(err).Msg("Resource initialization failed") @@ -94,23 +110,19 @@ func (o *Operator) Start() error { time.Sleep(initRetryWaitTime) } } - - //probe.SetReady() - o.run() - panic("unreachable") + o.runDeployments(stop) } -// run the operator. -// This registers a listener and waits until the process stops. -func (o *Operator) run() { - log := o.Dependencies.Log - - log.Info().Msgf("Running controller in namespace '%s'", o.Config.Namespace) - - go o.runDeployments() - go o.runLocalStorages() - - // Wait till done - ctx := context.TODO() - <-ctx.Done() +// onStartStorage starts the storage operator and run till given channel is closed. +func (o *Operator) onStartStorage(stop <-chan struct{}) { + for { + if err := o.initResourceIfNeeded(false, true); err == nil { + break + } else { + log.Error().Err(err).Msg("Resource initialization failed") + log.Info().Msgf("Retrying in %s...", initRetryWaitTime) + time.Sleep(initRetryWaitTime) + } + } + o.runLocalStorages(stop) } diff --git a/pkg/operator/operator_deployment.go b/pkg/operator/operator_deployment.go index 26dfed5c5..f1216ed03 100644 --- a/pkg/operator/operator_deployment.go +++ b/pkg/operator/operator_deployment.go @@ -23,7 +23,6 @@ package operator import ( - "context" "fmt" "github.com/pkg/errors" @@ -46,7 +45,7 @@ var ( // run the deployments part of the operator. // This registers a listener and waits until the process stops. -func (o *Operator) runDeployments() { +func (o *Operator) runDeployments(stop <-chan struct{}) { source := cache.NewListWatchFromClient( o.Dependencies.CRCli.DatabaseV1alpha().RESTClient(), api.ArangoDeploymentResourcePlural, @@ -59,9 +58,7 @@ func (o *Operator) runDeployments() { DeleteFunc: o.onDeleteArangoDeployment, }, cache.Indexers{}) - ctx := context.TODO() - // TODO: use workqueue to avoid blocking - informer.Run(ctx.Done()) + informer.Run(stop) } // onAddArangoDeployment deployment addition callback diff --git a/pkg/operator/operator_leader.go b/pkg/operator/operator_leader.go new file mode 100644 index 000000000..4c8eb625c --- /dev/null +++ b/pkg/operator/operator_leader.go @@ -0,0 +1,60 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package operator + +import ( + "time" + + "k8s.io/client-go/tools/leaderelection" + "k8s.io/client-go/tools/leaderelection/resourcelock" +) + +func (o *Operator) runLeaderElection(lockName string, onStart func(stop <-chan struct{})) { + namespace := o.Config.Namespace + kubecli := o.Dependencies.KubeCli + log := o.Dependencies.Log.With().Str("lock-name", lockName).Logger() + rl, err := resourcelock.New(resourcelock.EndpointsResourceLock, + namespace, + lockName, + kubecli.CoreV1(), + resourcelock.ResourceLockConfig{ + Identity: o.Config.ID, + EventRecorder: o.Dependencies.EventRecorder, + }) + if err != nil { + log.Fatal().Err(err).Msg("Failed to create resource lock") + } + + leaderelection.RunOrDie(leaderelection.LeaderElectionConfig{ + Lock: rl, + LeaseDuration: 15 * time.Second, + RenewDeadline: 10 * time.Second, + RetryPeriod: 2 * time.Second, + Callbacks: leaderelection.LeaderCallbacks{ + OnStartedLeading: onStart, + OnStoppedLeading: func() { + log.Info().Msg("Leader election lost") + }, + }, + }) +} diff --git a/pkg/operator/operator_local_storage.go b/pkg/operator/operator_local_storage.go index f9e4d7d91..33c532a84 100644 --- a/pkg/operator/operator_local_storage.go +++ b/pkg/operator/operator_local_storage.go @@ -23,7 +23,6 @@ package operator import ( - "context" "fmt" "github.com/pkg/errors" @@ -46,7 +45,7 @@ var ( // run the local storages part of the operator. // This registers a listener and waits until the process stops. -func (o *Operator) runLocalStorages() { +func (o *Operator) runLocalStorages(stop <-chan struct{}) { source := cache.NewListWatchFromClient( o.Dependencies.CRCli.StorageV1alpha().RESTClient(), api.ArangoLocalStorageResourcePlural, @@ -59,9 +58,7 @@ func (o *Operator) runLocalStorages() { DeleteFunc: o.onDeleteArangoLocalStorage, }, cache.Indexers{}) - ctx := context.TODO() - // TODO: use workqueue to avoid blocking - informer.Run(ctx.Done()) + informer.Run(stop) } // onAddArangoLocalStorage local storage addition callback diff --git a/tools/manifests/manifest_builder.go b/tools/manifests/manifest_builder.go index 6eca6afe6..0919e9482 100644 --- a/tools/manifests/manifest_builder.go +++ b/tools/manifests/manifest_builder.go @@ -38,40 +38,58 @@ import ( var ( options struct { - OutputFile string + OutputSuffix string TemplatesDir string - Namespace string - Image string - ImagePullPolicy string - ImageSHA256 bool - OperatorName string - RBAC bool + Namespace string + Image string + ImagePullPolicy string + ImageSHA256 bool + DeploymentOperatorName string + StorageOperatorName string + RBAC bool + } + deploymentTemplateNames = []string{ + "rbac.yaml", + "deployment.yaml", } - templateNames = []string{ + storageTemplateNames = []string{ "rbac.yaml", "deployment.yaml", } ) func init() { - pflag.StringVar(&options.OutputFile, "output", "manifests/arango-operator.yaml", "Path of the generated manifest file") + pflag.StringVar(&options.OutputSuffix, "output-suffix", "", "Suffix of the generated manifest files") pflag.StringVar(&options.TemplatesDir, "templates-dir", "manifests/templates", "Directory containing manifest templates") pflag.StringVar(&options.Namespace, "namespace", "default", "Namespace in which the operator will be deployed") pflag.StringVar(&options.Image, "image", "arangodb/arangodb-operator:latest", "Fully qualified image name of the ArangoDB operator") pflag.StringVar(&options.ImagePullPolicy, "image-pull-policy", "IfNotPresent", "Pull policy of the ArangoDB operator image") pflag.BoolVar(&options.ImageSHA256, "image-sha256", true, "Use SHA256 syntax for image") - pflag.StringVar(&options.OperatorName, "operator-name", "arango-operator", "Name of the ArangoDB operator deployment") + pflag.StringVar(&options.DeploymentOperatorName, "deployment-operator-name", "arango-deployment-operator", "Name of the ArangoDeployment operator deployment") + pflag.StringVar(&options.StorageOperatorName, "storage-operator-name", "arango-storage-operator", "Name of the ArangoLocalStorage operator deployment") pflag.BoolVar(&options.RBAC, "rbac", true, "Use role based access control") pflag.Parse() } +type TemplateOptions struct { + Image string + ImagePullPolicy string + RBAC bool + Deployment OperatorOptions + Storage OperatorOptions +} + +type OperatorOptions struct { + Namespace string + OperatorName string + ClusterRoleName string + ClusterRoleBindingName string +} + func main() { // Check options - if options.OutputFile == "" { - log.Fatal("--output not specified.") - } if options.Namespace == "" { log.Fatal("--namespace not specified.") } @@ -95,47 +113,55 @@ func main() { options.Image = strings.TrimSpace(string(result)) } + // Prepare templates to include + templateNameSet := map[string][]string{ + "deployment": deploymentTemplateNames, + "storage": storageTemplateNames, + } + // Process templates - templateOptions := struct { - Namespace string - OperatorName string - OperatorImage string - ImagePullPolicy string - ClusterRoleName string - ClusterRoleBindingName string - RBAC bool - }{ - Namespace: options.Namespace, - OperatorName: options.OperatorName, - OperatorImage: options.Image, - ImagePullPolicy: options.ImagePullPolicy, - ClusterRoleName: "arango-operator", - ClusterRoleBindingName: "arango-operator", - RBAC: options.RBAC, + templateOptions := TemplateOptions{ + Image: options.Image, + ImagePullPolicy: options.ImagePullPolicy, + RBAC: options.RBAC, + Deployment: OperatorOptions{ + Namespace: options.Namespace, + OperatorName: options.DeploymentOperatorName, + ClusterRoleName: "arango-deployment-operator", + ClusterRoleBindingName: "arango-deployment-operator", + }, + Storage: OperatorOptions{ + Namespace: options.Namespace, + OperatorName: options.StorageOperatorName, + ClusterRoleName: "arango-storage-operator", + ClusterRoleBindingName: "arango-storage-operator", + }, } - output := &bytes.Buffer{} - for i, name := range templateNames { - t, err := template.New(name).ParseFiles(filepath.Join(options.TemplatesDir, name)) + for group, templateNames := range templateNameSet { + output := &bytes.Buffer{} + for i, name := range templateNames { + t, err := template.New(name).ParseFiles(filepath.Join(options.TemplatesDir, group, name)) + if err != nil { + log.Fatalf("Failed to parse template %s: %v", name, err) + } + if i > 0 { + output.WriteString("\n---\n\n") + } + output.WriteString(fmt.Sprintf("## %s/%s\n", group, name)) + t.Execute(output, templateOptions) + output.WriteString("\n") + } + + // Save output + outputPath, err := filepath.Abs(filepath.Join("manifests", "arango-"+group+options.OutputSuffix+".yaml")) if err != nil { - log.Fatalf("Failed to parse template %s: %v", name, err) + log.Fatalf("Failed to get absolute output path: %v\n", err) } - if i > 0 { - output.WriteString("\n---\n\n") + if err := os.MkdirAll(filepath.Base(outputPath), 0755); err != nil { + log.Fatalf("Failed to create output directory: %v\n", err) + } + if err := ioutil.WriteFile(outputPath, output.Bytes(), 0644); err != nil { + log.Fatalf("Failed to write output file: %v\n", err) } - output.WriteString(fmt.Sprintf("## %s\n", name)) - t.Execute(output, templateOptions) - output.WriteString("\n") - } - - // Save output - outputPath, err := filepath.Abs(options.OutputFile) - if err != nil { - log.Fatalf("Failed to get absolute output path: %v\n", err) - } - if err := os.MkdirAll(filepath.Base(outputPath), 0755); err != nil { - log.Fatalf("Failed to create output directory: %v\n", err) - } - if err := ioutil.WriteFile(outputPath, output.Bytes(), 0644); err != nil { - log.Fatalf("Failed to write output file: %v\n", err) } } From 89be8d12558eb0597fa133bd0c1ad88432c9e1d2 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Mon, 19 Mar 2018 12:22:09 +0100 Subject: [PATCH 2/4] Removed ignores files --- manifests/arango-deployment-dev.yaml | 90 ---------------------------- manifests/arango-storage-dev.yaml | 86 -------------------------- 2 files changed, 176 deletions(-) delete mode 100644 manifests/arango-deployment-dev.yaml delete mode 100644 manifests/arango-storage-dev.yaml diff --git a/manifests/arango-deployment-dev.yaml b/manifests/arango-deployment-dev.yaml deleted file mode 100644 index e1f4c76ce..000000000 --- a/manifests/arango-deployment-dev.yaml +++ /dev/null @@ -1,90 +0,0 @@ -## deployment/rbac.yaml -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: arango-deployment-operator -rules: -- apiGroups: - - database.arangodb.com - resources: - - arangodeployments - verbs: - - "*" -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - "*" -- apiGroups: - - "" - resources: - - pods - - services - - endpoints - - persistentvolumeclaims - - events - - secrets - verbs: - - "*" -- apiGroups: - - apps - resources: - - deployments - verbs: - - "*" -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - ---- - -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: arango-deployment-operator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: arango-deployment-operator -subjects: -- kind: ServiceAccount - name: default - namespace: default - ---- - -## deployment/deployment.yaml - -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: arango-deployment-operator - namespace: default -spec: - replicas: 1 - template: - metadata: - labels: - name: arango-deployment-operator - spec: - containers: - - name: operator - imagePullPolicy: IfNotPresent - image: ewoutp/arangodb-operator@sha256:51a122f8b36cb26f181dd1ffdddae6cb3fb5a4def4709c3ea123744a557cbe08 - args: - - --operator.deployment - env: - - name: MY_POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: MY_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - diff --git a/manifests/arango-storage-dev.yaml b/manifests/arango-storage-dev.yaml deleted file mode 100644 index eaf7d3487..000000000 --- a/manifests/arango-storage-dev.yaml +++ /dev/null @@ -1,86 +0,0 @@ -## storage/rbac.yaml -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: arango-deployment-operator -rules: -- apiGroups: - - storage.arangodb.com - resources: - - arangolocalstorages - verbs: - - "*" -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - "*" -- apiGroups: - - "" - resources: - - persistentvolumes - - persistentvolumeclaims - - events - verbs: - - "*" -- apiGroups: - - apps - resources: - - daemonsets - verbs: - - "*" -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - "*" - ---- - -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: arango-deployment-operator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: arango-deployment-operator -subjects: -- kind: ServiceAccount - name: default - namespace: default - ---- - -## storage/deployment.yaml - -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: arango-storage-operator - namespace: default -spec: - replicas: 1 - template: - metadata: - labels: - name: arango-storage-operator - spec: - containers: - - name: operator - imagePullPolicy: IfNotPresent - image: ewoutp/arangodb-operator@sha256:51a122f8b36cb26f181dd1ffdddae6cb3fb5a4def4709c3ea123744a557cbe08 - args: - - --operator.storage - env: - - name: MY_POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: MY_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - From 5fa216cead9e421cf821c79d7cdfb851ba0ba7c6 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Mon, 19 Mar 2018 12:22:47 +0100 Subject: [PATCH 3/4] Updated ignore list --- manifests/.gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/manifests/.gitignore b/manifests/.gitignore index 163d5c8e5..10eae2548 100644 --- a/manifests/.gitignore +++ b/manifests/.gitignore @@ -1 +1,2 @@ -arango-operator-dev.yaml \ No newline at end of file +arango-deployment-dev.yaml +arango-storage-dev.yaml \ No newline at end of file From 3deff1b0566b37ca65abca89c5e8749883e0cf5a Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Mon, 19 Mar 2018 15:40:15 +0100 Subject: [PATCH 4/4] Fixed template --- manifests/templates/storage/rbac.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/manifests/templates/storage/rbac.yaml b/manifests/templates/storage/rbac.yaml index aea2a509e..2304045a6 100644 --- a/manifests/templates/storage/rbac.yaml +++ b/manifests/templates/storage/rbac.yaml @@ -3,7 +3,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: - name: {{ .Deployment.ClusterRoleName }} + name: {{ .Storage.ClusterRoleName }} rules: - apiGroups: - storage.arangodb.com @@ -43,14 +43,14 @@ rules: apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: - name: {{ .Deployment.ClusterRoleBindingName }} + name: {{ .Storage.ClusterRoleBindingName }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: {{ .Deployment.ClusterRoleName }} + name: {{ .Storage.ClusterRoleName }} subjects: - kind: ServiceAccount name: default - namespace: {{ .Deployment.Namespace }} + namespace: {{ .Storage.Namespace }} {{- end -}} \ No newline at end of file