Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add an ability to remove secrets and configmaps when database is gone #168

Merged
merged 3 commits into from
Jan 12, 2023
Merged

Add an ability to remove secrets and configmaps when database is gone #168

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3ec4945
Squashed commits to update username for commits
allanger Jan 11, 2023
39e6474
Restore crs
allanger Jan 11, 2023
be08047
Remove chart files
Jan 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ TMP_DIR=$$(mktemp -d) ;\
cd $$TMP_DIR ;\
go mod init tmp ;\
echo "Downloading $(2)" ;\
GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\
GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\
rm -rf $$TMP_DIR ;\
}
endef
1 change: 1 addition & 0 deletions api/v1alpha1/database_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ type DatabaseSpec struct {
ConnectionStringTemplate string `json:"connectionStringTemplate,omitempty"`
SecretsTemplates map[string]string `json:"secretsTemplates,omitempty"`
Postgres Postgres `json:"postgres,omitempty"`
Cleanup bool `json:"cleanup,omitempty"`
}

// Postgres struct should be used to provide resource that only applicable to postgres
Expand Down
9 changes: 5 additions & 4 deletions controllers/backup/cronjob.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ import (
// GCSBackupCron builds kubernetes cronjob object
// to create database backup regularly with defined schedule from dbcr
// this job will database dump and upload to google bucket storage for backup
func GCSBackupCron(conf *config.Config, dbcr *kciv1alpha1.Database) (*batchv1beta1.CronJob, error) {
func GCSBackupCron(conf *config.Config, dbcr *kciv1alpha1.Database, ownership []metav1.OwnerReference) (*batchv1beta1.CronJob, error) {
cronJobSpec, err := buildCronJobSpec(conf, dbcr)
if err != nil {
return nil, err
Expand All @@ -46,9 +46,10 @@ func GCSBackupCron(conf *config.Config, dbcr *kciv1alpha1.Database) (*batchv1bet
APIVersion: "batch",
},
ObjectMeta: metav1.ObjectMeta{
Name: dbcr.Namespace + "-" + dbcr.Name + "-" + "backup",
Namespace: dbcr.Namespace,
Labels: kci.BaseLabelBuilder(),
Name: dbcr.Namespace + "-" + dbcr.Name + "-" + "backup",
Namespace: dbcr.Namespace,
Labels: kci.BaseLabelBuilder(),
OwnerReferences: ownership,
},
Spec: cronJobSpec,
}, nil
Expand Down
45 changes: 41 additions & 4 deletions controllers/backup/cronjob_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,11 @@ import (
"github.com/stretchr/testify/assert"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func TestGCSBackupCronGsql(t *testing.T) {
ownership := []metav1.OwnerReference{}
dbcr := &kciv1alpha1.Database{}
dbcr.Namespace = "TestNS"
dbcr.Name = "TestDB"
Expand All @@ -43,15 +45,15 @@ func TestGCSBackupCronGsql(t *testing.T) {
conf := config.LoadConfig()

instance.Spec.Engine = "postgres"
funcCronObject, err := GCSBackupCron(&conf, dbcr)
funcCronObject, err := GCSBackupCron(&conf, dbcr, ownership)
if err != nil {
fmt.Print(err)
}

assert.Equal(t, "postgresbackupimage:latest", funcCronObject.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Image)

instance.Spec.Engine = "mysql"
funcCronObject, err = GCSBackupCron(&conf, dbcr)
funcCronObject, err = GCSBackupCron(&conf, dbcr, ownership)
if err != nil {
fmt.Print(err)
}
Expand All @@ -64,6 +66,7 @@ func TestGCSBackupCronGsql(t *testing.T) {
}

func TestGCSBackupCronGeneric(t *testing.T) {
ownership := []metav1.OwnerReference{}
dbcr := &kciv1alpha1.Database{}
dbcr.Namespace = "TestNS"
dbcr.Name = "TestDB"
Expand All @@ -78,15 +81,15 @@ func TestGCSBackupCronGeneric(t *testing.T) {
conf := config.LoadConfig()

instance.Spec.Engine = "postgres"
funcCronObject, err := GCSBackupCron(&conf, dbcr)
funcCronObject, err := GCSBackupCron(&conf, dbcr, ownership)
if err != nil {
fmt.Print(err)
}

assert.Equal(t, "postgresbackupimage:latest", funcCronObject.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Image)

instance.Spec.Engine = "mysql"
funcCronObject, err = GCSBackupCron(&conf, dbcr)
funcCronObject, err = GCSBackupCron(&conf, dbcr, ownership)
if err != nil {
fmt.Print(err)
}
Expand All @@ -96,8 +99,42 @@ func TestGCSBackupCronGeneric(t *testing.T) {
assert.Equal(t, "TestNS", funcCronObject.Namespace)
assert.Equal(t, "TestNS-TestDB-backup", funcCronObject.Name)
assert.Equal(t, "* * * * *", funcCronObject.Spec.Schedule)
assert.Equal(t, len(funcCronObject.OwnerReferences), 0, "Unexpected size of an OwnerReference")
}

func TestGCSBackupCronGenericWithOwnerReference(t *testing.T) {
ownership := []metav1.OwnerReference{}
ownership = append(ownership, metav1.OwnerReference{
APIVersion: "api-version",
Kind: "kind",
Name: "name",
UID: "uid",
})
dbcr := &kciv1alpha1.Database{}
dbcr.Namespace = "TestNS"
dbcr.Name = "TestDB"
instance := &kciv1alpha1.DbInstance{}
instance.Status.Info = map[string]string{"DB_CONN": "TestConnection", "DB_PORT": "1234"}
instance.Spec.Generic = &kciv1alpha1.GenericInstance{BackupHost: "slave.test"}
dbcr.Status.InstanceRef = instance
dbcr.Spec.Instance = "staging"
dbcr.Spec.Backup.Cron = "* * * * *"

os.Setenv("CONFIG_PATH", "./test/backup_config.yaml")
conf := config.LoadConfig()

instance.Spec.Engine = "postgres"
funcCronObject, err := GCSBackupCron(&conf, dbcr, ownership)
if err != nil {
fmt.Print(err)
}

assert.Equal(t, len(funcCronObject.OwnerReferences), 1, "Unexpected size of an OwnerReference")
assert.Equal(t, funcCronObject.OwnerReferences[0].APIVersion, ownership[0].APIVersion, "API Version in the OwnerReference is wrong")
assert.Equal(t, funcCronObject.OwnerReferences[0].Kind, ownership[0].Kind, "Kind in the OwnerReference is wrong")
assert.Equal(t, funcCronObject.OwnerReferences[0].Name, ownership[0].Name, "Name in the OwnerReference is wrong")
assert.Equal(t, funcCronObject.OwnerReferences[0].UID, ownership[0].UID, "UID in the OwnerReference is wrong")
}
func TestGetResourceRequirements(t *testing.T) {
os.Setenv("CONFIG_PATH", "./test/backup_config.yaml")
conf := config.LoadConfig()
Expand Down
88 changes: 45 additions & 43 deletions controllers/database_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,37 +179,47 @@ func (r *DatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c

// database status not true, process phase
if !dbcr.Status.Status {
ownership := []metav1.OwnerReference{}
if dbcr.Spec.Cleanup {
ownership = append(ownership, metav1.OwnerReference{
APIVersion: dbcr.APIVersion,
Kind: dbcr.Kind,
Name: dbcr.Name,
UID: dbcr.GetUID(),
},
)
}

phase := dbcr.Status.Phase
logrus.Infof("DB: namespace=%s, name=%s start %s", dbcr.Namespace, dbcr.Name, phase)

defer promDBsPhaseTime.WithLabelValues(phase).Observe(kci.TimeTrack(time.Now()))
err := r.createDatabase(ctx, dbcr)
err := r.createDatabase(ctx, dbcr, ownership)
if err != nil {
// when database creation failed, don't requeue request. to prevent exceeding api limit (ex: against google api)
return r.manageError(ctx, dbcr, err, false)
}

dbcr.Status.Phase = dbPhaseInstanceAccessSecret
err = r.createInstanceAccessSecret(ctx, dbcr)
if err != nil {

if err = r.createInstanceAccessSecret(ctx, dbcr, ownership); err != nil {
return r.manageError(ctx, dbcr, err, true)
}
dbcr.Status.Phase = dbPhaseProxy
err = r.createProxy(ctx, dbcr)
err = r.createProxy(ctx, dbcr, ownership)
if err != nil {
return r.manageError(ctx, dbcr, err, true)
}
dbcr.Status.Phase = dbPhaseSecretsTemplating
err = r.createTemplatedSecrets(ctx, dbcr)
if err != nil {
if err = r.createTemplatedSecrets(ctx, dbcr, ownership); err != nil {
return r.manageError(ctx, dbcr, err, true)
}
dbcr.Status.Phase = dbPhaseConfigMap
err = r.createInfoConfigMap(ctx, dbcr)
if err != nil {
if err = r.createInfoConfigMap(ctx, dbcr, ownership); err != nil {
return r.manageError(ctx, dbcr, err, true)
}
dbcr.Status.Phase = dbPhaseBackupJob
err = r.createBackupJob(ctx, dbcr)
err = r.createBackupJob(ctx, dbcr, ownership)
if err != nil {
return r.manageError(ctx, dbcr, err, true)
}
Expand Down Expand Up @@ -278,7 +288,7 @@ func (r *DatabaseReconciler) initialize(ctx context.Context, dbcr *kciv1alpha1.D
}

// createDatabase secret, actual database using admin secret
func (r *DatabaseReconciler) createDatabase(ctx context.Context, dbcr *kciv1alpha1.Database) error {
func (r *DatabaseReconciler) createDatabase(ctx context.Context, dbcr *kciv1alpha1.Database, ownership []metav1.OwnerReference) error {
databaseSecret, err := r.getDatabaseSecret(ctx, dbcr)
if err != nil {
if k8serrors.IsNotFound(err) {
Expand All @@ -287,7 +297,7 @@ func (r *DatabaseReconciler) createDatabase(ctx context.Context, dbcr *kciv1alph
logrus.Errorf("can not generate credentials for database - %s", err)
return err
}
newDatabaseSecret := kci.SecretBuilder(dbcr.Spec.SecretName, dbcr.Namespace, secretData)
newDatabaseSecret := kci.SecretBuilder(dbcr.Spec.SecretName, dbcr.Namespace, secretData, ownership)
err = r.Create(ctx, newDatabaseSecret)
if err != nil {
// failed to create secret
Expand Down Expand Up @@ -388,7 +398,7 @@ func (r *DatabaseReconciler) deleteDatabase(ctx context.Context, dbcr *kciv1alph
return nil
}

func (r *DatabaseReconciler) createInstanceAccessSecret(ctx context.Context, dbcr *kciv1alpha1.Database) error {
func (r *DatabaseReconciler) createInstanceAccessSecret(ctx context.Context, dbcr *kciv1alpha1.Database, ownership []metav1.OwnerReference) error {
if backend, _ := dbcr.GetBackendType(); backend != "google" {
logrus.Debugf("DB: namespace=%s, name=%s %s doesn't need instance access secret skipping...", dbcr.Namespace, dbcr.Name, backend)
return nil
Expand Down Expand Up @@ -422,7 +432,7 @@ func (r *DatabaseReconciler) createInstanceAccessSecret(ctx context.Context, dbc
secretData[credFile] = data

newName := dbcr.InstanceAccessSecretName()
newSecret := kci.SecretBuilder(newName, dbcr.GetNamespace(), secretData)
newSecret := kci.SecretBuilder(newName, dbcr.GetNamespace(), secretData, ownership)

err = r.Create(ctx, newSecret)
if err != nil {
Expand All @@ -442,7 +452,7 @@ func (r *DatabaseReconciler) createInstanceAccessSecret(ctx context.Context, dbc
return nil
}

func (r *DatabaseReconciler) createProxy(ctx context.Context, dbcr *kciv1alpha1.Database) error {
func (r *DatabaseReconciler) createProxy(ctx context.Context, dbcr *kciv1alpha1.Database, ownership []metav1.OwnerReference) error {
backend, _ := dbcr.GetBackendType()
if backend == "generic" {
logrus.Infof("DB: namespace=%s, name=%s %s proxy creation is not yet implemented skipping...", dbcr.Namespace, dbcr.Name, backend)
Expand All @@ -455,7 +465,7 @@ func (r *DatabaseReconciler) createProxy(ctx context.Context, dbcr *kciv1alpha1.
}

// create proxy configmap
cm, err := proxy.BuildConfigmap(proxyInterface)
cm, err := proxy.BuildConfigmap(proxyInterface, ownership)
if err != nil {
return err
}
Expand All @@ -478,7 +488,7 @@ func (r *DatabaseReconciler) createProxy(ctx context.Context, dbcr *kciv1alpha1.
}

// create proxy deployment
deploy, err := proxy.BuildDeployment(proxyInterface)
deploy, err := proxy.BuildDeployment(proxyInterface, ownership)
if err != nil {
return err
}
Expand All @@ -499,7 +509,7 @@ func (r *DatabaseReconciler) createProxy(ctx context.Context, dbcr *kciv1alpha1.
}

// create proxy service
svc, err := proxy.BuildService(proxyInterface)
svc, err := proxy.BuildService(proxyInterface, ownership)
if err != nil {
return err
}
Expand Down Expand Up @@ -533,7 +543,7 @@ func (r *DatabaseReconciler) createProxy(ctx context.Context, dbcr *kciv1alpha1.

if isMonitoringEnabled && inCrdList(crdList, "servicemonitors.monitoring.coreos.com") {
// create proxy PromServiceMonitor
promSvcMon, err := proxy.BuildServiceMonitor(proxyInterface)
promSvcMon, err := proxy.BuildServiceMonitor(proxyInterface, ownership)
if err != nil {
return err
}
Expand Down Expand Up @@ -567,15 +577,17 @@ func (r *DatabaseReconciler) createProxy(ctx context.Context, dbcr *kciv1alpha1.
return nil
}

func (r *DatabaseReconciler) createTemplatedSecrets(ctx context.Context, dbcr *kciv1alpha1.Database) error {
func (r *DatabaseReconciler) createTemplatedSecrets(ctx context.Context, dbcr *kciv1alpha1.Database, ownership []metav1.OwnerReference) error {
// First of all the password should be taken from secret because it's not stored anywhere else
databaseSecret, err := r.getDatabaseSecret(ctx, dbcr)
if err != nil {
return err
}
// Then parse the secret to get the password
// Connection stirng is deprecated and will be removed soon. So this switch is temporary.
// Connection string is deprecated and will be removed soon. So this switch is temporary.
// Once connection string is removed, the switch and the following if condition are gone
// Connection String doesn't support the cleaning up feature, so the secret with a connection
// string won't be removed after a db resource is removed.
useLegacyConnectionString := false
switch {
case len(dbcr.Spec.ConnectionStringTemplate) > 0 && len(dbcr.Spec.SecretsTemplates) > 0:
Expand Down Expand Up @@ -610,29 +622,31 @@ func (r *DatabaseReconciler) createTemplatedSecrets(ctx context.Context, dbcr *k
}
logrus.Debugf("DB: namespace=%s, name=%s updating credentials secret", dbcr.Namespace, dbcr.Name)
newSecret := addConnectionStringToSecret(dbcr, databaseSecret.Data, dbConnectionString)
return r.Update(ctx, newSecret, &client.UpdateOptions{})
if err = r.Update(ctx, newSecret, &client.UpdateOptions{}); err != nil {
return err
}
return nil
}

dbSecrets, err := generateTemplatedSecrets(dbcr, databaseCred)
if err != nil {
return err
}
// Adding values
newSecret := fillTemplatedSecretData(dbcr, databaseSecret.Data, dbSecrets)
err = r.Update(ctx, newSecret, &client.UpdateOptions{})
if err != nil {
newSecret := fillTemplatedSecretData(dbcr, databaseSecret.Data, dbSecrets, ownership)
if err = r.Update(ctx, newSecret, &client.UpdateOptions{}); err != nil {
return err
}
newSecret = removeObsoleteSecret(dbcr, databaseSecret.Data, dbSecrets)
err = r.Update(ctx, newSecret, &client.UpdateOptions{})
if err != nil {

newSecret = removeObsoleteSecret(dbcr, databaseSecret.Data, dbSecrets, ownership)
if err = r.Update(ctx, newSecret, &client.UpdateOptions{}); err != nil {
return err
}

return nil
}

func (r *DatabaseReconciler) createInfoConfigMap(ctx context.Context, dbcr *kciv1alpha1.Database) error {
func (r *DatabaseReconciler) createInfoConfigMap(ctx context.Context, dbcr *kciv1alpha1.Database, ownership []metav1.OwnerReference) error {
instance, err := dbcr.GetInstanceRef()
if err != nil {
return err
Expand All @@ -645,19 +659,7 @@ func (r *DatabaseReconciler) createInfoConfigMap(ctx context.Context, dbcr *kciv
info["DB_HOST"] = proxyStatus.ServiceName
info["DB_PORT"] = strconv.FormatInt(int64(proxyStatus.SQLPort), 10)
}

databaseConfigResource := &corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{
Kind: "ConfigMap",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Namespace: dbcr.Namespace,
Name: dbcr.Spec.SecretName,
Labels: kci.BaseLabelBuilder(),
},
Data: info,
}
databaseConfigResource := kci.ConfigMapBuilder(dbcr.Spec.SecretName, dbcr.Namespace, info, ownership)

err = r.Create(ctx, databaseConfigResource)
if err != nil {
Expand All @@ -678,13 +680,13 @@ func (r *DatabaseReconciler) createInfoConfigMap(ctx context.Context, dbcr *kciv
return nil
}

func (r *DatabaseReconciler) createBackupJob(ctx context.Context, dbcr *kciv1alpha1.Database) error {
func (r *DatabaseReconciler) createBackupJob(ctx context.Context, dbcr *kciv1alpha1.Database, ownership []metav1.OwnerReference) error {
if !dbcr.Spec.Backup.Enable {
// if not enabled, skip
return nil
}

cronjob, err := backup.GCSBackupCron(r.Conf, dbcr)
cronjob, err := backup.GCSBackupCron(r.Conf, dbcr, ownership)
if err != nil {
return err
}
Expand Down
Loading