Change the way the configuration service is configured (#10)

* Add improvements to the service

This commit changes the way the configuration service URL is configured
from the config.yaml of the service to a configuration in the helm
chart. Additionally it:

  * adds the scapability to display kubernetes logs within the keptn bridge
  * makes the kubernetes jub run as non-root
  * adds some more logging to the service

* fix deploy/service.yaml to work when deployed in the execution plane. Cleanup handling of env variables required by initcontainer. Fix tests and golint errors.

* fix goling error, add comment to exported EventHandler

* update reviewdog go version from 1.13 to 1.16

* fix gofmt and golint errors

* Add comments in config.go the linting indicated

* Fix reviewdog config for go vet

Co-authored-by: Dominik Augustin <[email protected]>

Author: didiladi

.github/workflows/reviewdog.yml

@@ -8,7 +8,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.13
+          go-version: 1.16
         id: go
       - name: Check out code.
         uses: actions/checkout@v1

.reviewdog.yml

@@ -9,4 +9,4 @@ runner:
     errorformat:
       - "%f:%l %m"
   govet:
-    cmd: go vet -all .
+    cmd: go vet ./...

cmd/keptn-generic-job-service-initcontainer/main.go

@@ -13,23 +13,19 @@ import (
 )
 
 type envConfig struct {
-	// Port on which to listen for cloudevents
-	Port int `envconfig:"RCV_PORT" default:"8080"`
-	// Path to which cloudevents are sent
-	Path string `envconfig:"RCV_PATH" default:"/"`
 	// Whether we are running locally (e.g., for testing) or on production
 	Env string `envconfig:"ENV" default:"local"`
 	// URL of the Keptn configuration service (this is where we can fetch files from the config repo)
-	ConfigurationServiceUrl string `envconfig:"CONFIGURATION_SERVICE" default:""`
+	ConfigurationServiceURL string `envconfig:"CONFIGURATION_SERVICE" required:"true"`
+	// The token of the keptn API
+	KeptnAPIToken string `envconfig:"KEPTN_API_TOKEN" required:"true"`
 	// The keptn project contained in the initial cloud event
 	Project string `envconfig:"KEPTN_PROJECT" required:"true"`
 	// The keptn stage contained in the initial cloud event
 	Stage string `envconfig:"KEPTN_STAGE" required:"true"`
 	// The keptn service contained in the initial cloud event
 	Service string `envconfig:"KEPTN_SERVICE" required:"true"`
 	// The keptn service contained in the initial cloud event
-	ApiToken string `envconfig:"KEPTN_API_TOKEN" required:"false"`
-	// The name of the config action which triggered the init container run
 	Action string `envconfig:"JOB_ACTION" required:"true"`
 	// The name of the config task which triggered the init container run
 	Task string `envconfig:"JOB_TASK" required:"true"`
@@ -45,11 +41,11 @@ func main() {
 	fs := afero.NewOsFs()
 
 	var resourceHandler *api.ResourceHandler
-	if env.ApiToken != "" {
-		configurationServiceUrl, _ := url.Parse(env.ConfigurationServiceUrl)
-		resourceHandler = api.NewAuthenticatedResourceHandler(configurationServiceUrl.String(), env.ApiToken, "x-token", nil, configurationServiceUrl.Scheme)
+	if env.KeptnAPIToken != "" { // gets set as empty string from the generic-job-service if the env variable is not set
+		configurationServiceURL, _ := url.Parse(env.ConfigurationServiceURL)
+		resourceHandler = api.NewAuthenticatedResourceHandler(configurationServiceURL.String(), env.KeptnAPIToken, "x-token", nil, configurationServiceURL.Scheme)
 	} else {
-		resourceHandler = api.NewResourceHandler(env.ConfigurationServiceUrl)
+		resourceHandler = api.NewResourceHandler(env.ConfigurationServiceURL)
 	}
 
 	useLocalFileSystem := false
@@ -60,7 +56,7 @@ func main() {
 		useLocalFileSystem = true
 	}
 
-	configService := keptn.NewKeptnConfigService(useLocalFileSystem, env.Project, env.Stage, env.Service, resourceHandler)
+	configService := keptn.NewConfigService(useLocalFileSystem, env.Project, env.Stage, env.Service, resourceHandler)
 
 	err := file.MountFiles(env.Action, env.Task, fs, configService)
 	if err != nil {

cmd/keptn-generic-job-service/main.go

@@ -3,19 +3,19 @@ package main
 import (
 	"context"
 	"didiladi/keptn-generic-job-service/pkg/eventhandler"
-	"encoding/json"
 	"errors"
 	"log"
 	"os"
 	"strings"
 
 	cloudevents "github.com/cloudevents/sdk-go/v2" // make sure to use v2 cloudevents here
 	"github.com/kelseyhightower/envconfig"
-	keptn "github.com/keptn/go-utils/pkg/lib/keptn"
+	"github.com/keptn/go-utils/pkg/lib/keptn"
 	keptnv2 "github.com/keptn/go-utils/pkg/lib/v0_2_0"
 )
 
 var keptnOptions = keptn.KeptnOpts{}
+var env envConfig
 
 type envConfig struct {
 	// Port on which to listen for cloudevents
@@ -25,7 +25,13 @@ type envConfig struct {
 	// Whether we are running locally (e.g., for testing) or on production
 	Env string `envconfig:"ENV" default:"local"`
 	// URL of the Keptn configuration service (this is where we can fetch files from the config repo)
-	ConfigurationServiceUrl string `envconfig:"CONFIGURATION_SERVICE" default:""`
+	ConfigurationServiceURL string `envconfig:"CONFIGURATION_SERVICE" default:""`
+	// The endpoint of the keptn configuration service API
+	InitContainerConfigurationServiceAPIEndpoint string `envconfig:"INIT_CONTAINER_CONFIGURATION_SERVICE_API_ENDPOINT" required:"true"`
+	// The k8s namespace the job will run in
+	JobNamespace string `envconfig:"JOB_NAMESPACE" required:"true"`
+	// The token of the keptn API
+	KeptnAPIToken string `envconfig:"KEPTN_API_TOKEN"`
 }
 
 // ServiceName specifies the current services name (e.g., used as source when sending CloudEvents)
@@ -68,16 +74,18 @@ func processKeptnCloudEvent(ctx context.Context, event cloudevents.Event) error
 		log.Printf("failed to convert incoming cloudevent to event data: %v", err)
 	}
 
-	var eventDataAsInterface interface{}
-	//err = event.DataAs(eventDataAsInterface)
-	err = json.Unmarshal(event.Data(), &eventDataAsInterface)
-	//err = parseKeptnCloudEventPayload(event, eventDataAsInterface)
-	if err != nil {
-		log.Printf("failed to convert incoming cloudevent: %v", err)
+	eventHandler := &eventhandler.EventHandler{
+		Keptn:        myKeptn,
+		Event:        event,
+		EventData:    eventData,
+		ServiceName:  ServiceName,
+		JobNamespace: env.JobNamespace,
+		InitContainerConfigurationServiceAPIEndpoint: env.InitContainerConfigurationServiceAPIEndpoint,
+		KeptnAPIToken: env.KeptnAPIToken,
 	}
 
 	// prevent duplicate events - https://github.com/keptn/keptn/issues/3888
-	go eventhandler.HandleEvent(myKeptn, event, eventDataAsInterface, eventData, ServiceName)
+	go eventHandler.HandleEvent()
 
 	return nil
 }
@@ -90,7 +98,6 @@ func processKeptnCloudEvent(ctx context.Context, event cloudevents.Event) error
  * env=runlocal   -> will fetch resources from local drive instead of configuration service
  */
 func main() {
-	var env envConfig
 	if err := envconfig.Process("", &env); err != nil {
 		log.Fatalf("Failed to process env var: %s", err)
 	}
@@ -109,12 +116,7 @@ func _main(args []string, env envConfig) int {
 		keptnOptions.UseLocalFileSystem = true
 	}
 
-	_, available := os.LookupEnv("JOB_NAMESPACE")
-	if !available {
-		log.Fatalf("JOB_NAMESPACE was not available. Please set it as env var")
-	}
-
-	keptnOptions.ConfigurationServiceURL = env.ConfigurationServiceUrl
+	keptnOptions.ConfigurationServiceURL = env.ConfigurationServiceURL
 
 	log.Println("Starting keptn-generic-job-service...")
 	log.Printf("    on Port = %d; Path=%s", env.Port, env.Path)

deploy/service.yaml

@@ -21,6 +21,8 @@ spec:
           ports:
             - containerPort: 8080
           env:
+            - name: INIT_CONTAINER_CONFIGURATION_SERVICE_API_ENDPOINT
+              value: "http://configuration-service:8080"
             - name: CONFIGURATION_SERVICE
               value: 'http://configuration-service:8080'
             - name: JOB_NAMESPACE
@@ -92,6 +94,7 @@ rules:
       - ""
     resources:
       - "pods"
+      - "pods/log"
       - "persistentvolumeclaims"
       - "jobs"
     verbs:

go.mod

@@ -3,6 +3,8 @@ module didiladi/keptn-generic-job-service
 go 1.16
 
 require (
+	cloud.google.com/go v0.82.0 // indirect
+	cloud.google.com/go/datastore v1.5.0 // indirect
 	github.com/PaesslerAG/jsonpath v0.1.1
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
 	github.com/cloudevents/sdk-go/v2 v2.4.1
@@ -12,23 +14,33 @@ require (
 	github.com/go-openapi/strfmt v0.20.1 // indirect
 	github.com/go-openapi/swag v0.19.15 // indirect
 	github.com/go-openapi/validate v0.20.2 // indirect
-	github.com/golang/mock v1.4.4
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/mock v1.5.0
+	github.com/google/go-github/v29 v29.0.3 // indirect
+	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/googleapis/gnostic v0.5.1 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.0 // indirect
 	github.com/json-iterator/go v1.1.11 // indirect
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/keptn/go-utils v0.8.3
 	github.com/keptn/kubernetes-utils v0.8.1
 	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-shellwords v1.0.11 // indirect
+	github.com/reviewdog/errorformat v0.0.0-20210517100703-fd739bda5dda // indirect
+	github.com/reviewdog/reviewdog v0.11.0 // indirect
 	github.com/spf13/afero v1.2.2
+	github.com/xanzy/go-gitlab v0.50.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.16.0 // indirect
-	golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 // indirect
-	golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c // indirect
-	golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 // indirect
+	golang.org/x/build v0.0.0-20210518220400-d0819edf598a // indirect
+	golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a // indirect
+	golang.org/x/net v0.0.0-20210510120150-4163338589ed // indirect
 	golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/api v0.47.0 // indirect
+	google.golang.org/genproto v0.0.0-20210518161634-ec7691c0a37d // indirect
+	google.golang.org/grpc v1.38.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0
 	gotest.tools v2.2.0+incompatible
 	k8s.io/api v0.21.0