From c36a9fa01b193d1ea94cd69c0544de549dbc93e1 Mon Sep 17 00:00:00 2001 From: Maysun J Faisal Date: Tue, 5 Dec 2023 16:33:03 -0500 Subject: [PATCH 1/2] Update mock to test devfile with private parent Signed-off-by: Maysun J Faisal --- .gitignore | 1 + pkg/devfile/parser/parse_test.go | 2 +- pkg/devfile/parser/reader_test.go | 6 +- pkg/devfile/parser/util/mock.go | 48 +++++++-- pkg/devfile/parser/util/utils_test.go | 8 +- pkg/util/mock.go | 139 ++++++++++++++++++++++++-- 6 files changed, 177 insertions(+), 27 deletions(-) diff --git a/.gitignore b/.gitignore index b98f9699..1396c90a 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ main # File created running tests tests/**/tmp/ tests/v2/lib-test-coverage.* +*resource.file* # Mac related files diff --git a/pkg/devfile/parser/parse_test.go b/pkg/devfile/parser/parse_test.go index 5e4d83d0..ab95c806 100644 --- a/pkg/devfile/parser/parse_test.go +++ b/pkg/devfile/parser/parse_test.go @@ -4448,7 +4448,7 @@ func Test_parseFromURI_GitProviders(t *testing.T) { tt.devfileUtilsClient.GitTestToken = tt.token tt.devfileUtilsClient.MockGitURL = util.MockGitUrl(*tt.gitUrl) - got, err := parseFromURI(tt.importReference, curDevfileContext, &resolutionContextTree{}, resolverTools{downloadGitResources: tt.downloadGitResources, devfileUtilsClient: tt.devfileUtilsClient}) + got, err := parseFromURI(tt.importReference, curDevfileContext, &resolutionContextTree{}, resolverTools{downloadGitResources: tt.downloadGitResources, devfileUtilsClient: &tt.devfileUtilsClient}) // validate even if we want an error; check that no files are copied to destDir validateGitResourceFunctions(t, tt.wantResources, tt.wantResourceContent, destDir) diff --git a/pkg/devfile/parser/reader_test.go b/pkg/devfile/parser/reader_test.go index 2000154d..53d110e9 100644 --- a/pkg/devfile/parser/reader_test.go +++ b/pkg/devfile/parser/reader_test.go @@ -198,7 +198,7 @@ func TestReadAndParseKubernetesYaml(t *testing.T) { URL: "http://" + serverIP, }, fs: nil, - devfileUtilsClient: parserUtil.MockDevfileUtilsClient{DownloadOptions: util.MockDownloadOptions{MockFile: string(data)}, MockGitURL: util.MockGitUrl{Host: "http://github.com"}}, + devfileUtilsClient: &parserUtil.MockDevfileUtilsClient{DownloadOptions: util.MockDownloadOptions{MockFile: string(data)}, MockGitURL: util.MockGitUrl{Host: "http://github.com"}}, wantDeploymentNames: []string{"deploy-sample", "deploy-sample-2"}, wantServiceNames: []string{"service-sample", "service-sample-2"}, wantRouteNames: []string{"route-sample", "route-sample-2"}, @@ -212,7 +212,7 @@ func TestReadAndParseKubernetesYaml(t *testing.T) { Token: "valid-token", }, fs: nil, - devfileUtilsClient: parserUtil.MockDevfileUtilsClient{DownloadOptions: util.MockDownloadOptions{MockFile: string(data)}, MockGitURL: util.MockGitUrl{Host: "http://github.com"}, GitTestToken: "valid-token"}, + devfileUtilsClient: &parserUtil.MockDevfileUtilsClient{DownloadOptions: util.MockDownloadOptions{MockFile: string(data)}, MockGitURL: util.MockGitUrl{Host: "http://github.com"}, GitTestToken: "valid-token"}, wantDeploymentNames: []string{"deploy-sample", "deploy-sample-2"}, wantServiceNames: []string{"service-sample", "service-sample-2"}, wantRouteNames: []string{"route-sample", "route-sample-2"}, @@ -226,7 +226,7 @@ func TestReadAndParseKubernetesYaml(t *testing.T) { Token: "invalid-token", }, fs: nil, - devfileUtilsClient: parserUtil.MockDevfileUtilsClient{DownloadOptions: util.MockDownloadOptions{MockFile: string(data)}, MockGitURL: util.MockGitUrl{Host: "http://github.com"}, GitTestToken: "invalid-token"}, + devfileUtilsClient: &parserUtil.MockDevfileUtilsClient{DownloadOptions: util.MockDownloadOptions{MockFile: string(data)}, MockGitURL: util.MockGitUrl{Host: "http://github.com"}, GitTestToken: "invalid-token"}, wantErr: true, }, } diff --git a/pkg/devfile/parser/util/mock.go b/pkg/devfile/parser/util/mock.go index 0f29069a..ec6bd24f 100644 --- a/pkg/devfile/parser/util/mock.go +++ b/pkg/devfile/parser/util/mock.go @@ -42,17 +42,14 @@ func NewMockDevfileUtilsClient() MockDevfileUtilsClient { return MockDevfileUtilsClient{} } -func (gc MockDevfileUtilsClient) DownloadInMemory(params util.HTTPRequestParams) ([]byte, error) { +func (gc *MockDevfileUtilsClient) DownloadInMemory(params util.HTTPRequestParams) ([]byte, error) { var httpClient = &http.Client{Transport: &http.Transport{ ResponseHeaderTimeout: util.HTTPRequestResponseTimeout, }, Timeout: util.HTTPRequestResponseTimeout} - var mockGitUrl util.MockGitUrl - if gc.MockGitURL.Host != "" { if util.IsGitProviderRepo(gc.MockGitURL.Host) { - mockGitUrl = gc.MockGitURL - mockGitUrl.Token = gc.GitTestToken + gc.MockGitURL.Token = gc.GitTestToken } } else if params.URL != "" { // Not all clients have the ability to pass in mock data @@ -60,16 +57,51 @@ func (gc MockDevfileUtilsClient) DownloadInMemory(params util.HTTPRequestParams) // and mock the output if util.IsGitProviderRepo(params.URL) { gc.MockGitURL.Host = params.URL - mockGitUrl = gc.MockGitURL - mockGitUrl.Token = params.Token + gc.MockGitURL.Token = params.Token } } - return mockGitUrl.DownloadInMemoryWithClient(params, httpClient, gc.DownloadOptions) + if gc.DownloadOptions.MockParent == nil { + gc.DownloadOptions.MockParent = &util.MockParent{} + } + + file, err := gc.MockGitURL.DownloadInMemoryWithClient(params, httpClient, gc.DownloadOptions) + + if gc.DownloadOptions.MockParent != nil && gc.DownloadOptions.MockParent.IsMainDevfileDownloaded && gc.DownloadOptions.MockParent.IsParentDevfileDownloaded { + // Since gc is a pointer, if both the main and parent devfiles are downloaded, reset the flag. + // So that other tests can use the Mock Parent Devfile download if required. + gc.DownloadOptions.MockParent.IsMainDevfileDownloaded = false + gc.DownloadOptions.MockParent.IsParentDevfileDownloaded = false + } + + if gc.MockGitURL.Host != "" && params.URL != "" { + // Since gc is a pointer, reset the mock data if both the URL and Host are present + gc.MockGitURL.Host = "" + gc.MockGitURL.Token = "" + } + + return file, err } func (gc MockDevfileUtilsClient) DownloadGitRepoResources(url string, destDir string, token string) error { + // if mock data is unavailable as certain clients cant provide mock data + // then adapt and create mock data from actual params + if gc.ParentURLAlias == "" { + gc.ParentURLAlias = url + gc.MockGitURL.IsFile = true + gc.MockGitURL.Revision = "main" + gc.MockGitURL.Path = OutputDevfileYamlPath + gc.MockGitURL.Host = "github.com" + gc.MockGitURL.Protocol = "https" + gc.MockGitURL.Owner = "devfile" + gc.MockGitURL.Repo = "library" + } + + if gc.GitTestToken == "" { + gc.GitTestToken = token + } + //the url parameter that gets passed in will be the localhost IP of the test server, so it will fail all the validation checks. We will use the global testURL variable instead //skip the Git Provider check since it'll fail if util.IsGitProviderRepo(gc.ParentURLAlias) { diff --git a/pkg/devfile/parser/util/utils_test.go b/pkg/devfile/parser/util/utils_test.go index e8cf958d..b4b00e33 100644 --- a/pkg/devfile/parser/util/utils_test.go +++ b/pkg/devfile/parser/util/utils_test.go @@ -90,25 +90,25 @@ func TestDownloadInMemoryClient(t *testing.T) { }, { name: "Case 6: Input url is valid with a mock client, dont use mock data during invocation", - client: MockDevfileUtilsClient{}, + client: &MockDevfileUtilsClient{}, url: server.URL, want: []byte{79, 75}, }, { name: "Case 7: Input url is valid with a mock client and mock token", - client: MockDevfileUtilsClient{MockGitURL: util.MockGitUrl{Host: "https://github.com/devfile/library/blob/main/devfile.yaml"}, GitTestToken: "valid-token", DownloadOptions: util.MockDownloadOptions{MockFile: "OK"}}, + client: &MockDevfileUtilsClient{MockGitURL: util.MockGitUrl{Host: "https://github.com/devfile/library/blob/main/devfile.yaml"}, GitTestToken: "valid-token", DownloadOptions: util.MockDownloadOptions{MockFile: "OK"}}, url: "https://github.com/devfile/library/blob/main/devfile.yaml", want: []byte{79, 75}, }, { name: "Case 8: Public Github repo, with invalid token ", - client: MockDevfileUtilsClient{MockGitURL: util.MockGitUrl{Host: "https://github.com/devfile/library/blob/main/devfile.yaml"}, GitTestToken: "invalid-token"}, + client: &MockDevfileUtilsClient{MockGitURL: util.MockGitUrl{Host: "https://github.com/devfile/library/blob/main/devfile.yaml"}, GitTestToken: "invalid-token"}, url: "https://github.com/devfile/library/blob/main/devfile.yaml", wantErr: "failed to retrieve https://github.com/devfile/library/blob/main/devfile.yaml", }, { name: "Case 9: Input github url is valid with a mock client, dont use mock data during invocation", - client: MockDevfileUtilsClient{}, + client: &MockDevfileUtilsClient{}, url: "https://raw.githubusercontent.com/maysunfaisal/OK/main/OK.txt", want: []byte{79, 75}, }, diff --git a/pkg/util/mock.go b/pkg/util/mock.go index bc7d8c77..fb30ea61 100644 --- a/pkg/util/mock.go +++ b/pkg/util/mock.go @@ -37,6 +37,12 @@ type MockDownloadOptions struct { MockDevfile bool MockDockerfile bool MockFile string + MockParent *MockParent +} + +type MockParent struct { + IsMainDevfileDownloaded bool + IsParentDevfileDownloaded bool } func (m *MockGitUrl) GetToken() string { @@ -57,6 +63,8 @@ var mockExecute = func(baseDir string, cmd CommandType, args ...string) ([]byte, // private repository if hasPassword { switch password { + case "parent-devfile": + fallthrough case "valid-token": _, err := resourceFile.WriteString("private repo\n") if err != nil { @@ -147,18 +155,8 @@ metadata: tags: - Go version: 1.0.0 -starterProjects: - - name: go-starter - git: - checkoutFrom: - revision: main - remotes: - origin: https://github.com/devfile-samples/devfile-stack-go.git components: - container: - endpoints: - - name: http - targetPort: 8080 image: golang:latest memoryLimit: 1024Mi mountSources: true @@ -190,7 +188,6 @@ commands: commandLine: GOCACHE=/project/.cache go build main.go component: runtime group: - isDefault: true kind: build workingDir: /project id: build @@ -198,7 +195,78 @@ commands: commandLine: ./main component: runtime group: + kind: run + workingDir: /project + id: run + - id: build-image + apply: + component: image-build + - id: deployk8s + apply: + component: kubernetes-deploy + - id: deploy + composite: + commands: + - build-image + - deployk8s + group: + kind: deploy isDefault: true +` + +var mockDevfileWithParentRef = ` +schemaVersion: 2.2.0 +metadata: + displayName: Go Mock Runtime + icon: https://raw.githubusercontent.com/devfile-samples/devfile-stack-icons/main/golang.svg + language: go + name: go + projectType: go + tags: + - Go + version: 1.0.0 +parent: + uri: https://github.com/private-url-devfile +components: + - container: + image: golang:latest + memoryLimit: 1024Mi + mountSources: true + sourceMapping: /project + name: runtime + - name: image-build + image: + imageName: go-image:latest + dockerfile: + uri: docker/Dockerfile + buildContext: . + rootRequired: false + - name: kubernetes-deploy + kubernetes: + inlined: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + creationTimestamp: null + labels: + test: test + name: deploy-sample + endpoints: + - name: http-8081 + targetPort: 8081 + path: / +commands: + - exec: + commandLine: GOCACHE=/project/.cache go build main.go + component: runtime + group: + kind: build + workingDir: /project + id: build + - exec: + commandLine: ./main + component: runtime + group: kind: run workingDir: /project id: run @@ -218,6 +286,45 @@ commands: isDefault: true ` +var mockParentDevfile = ` +schemaVersion: 2.2.0 +metadata: + displayName: Go Mock Parent + language: go + name: goparent + projectType: go + tags: + - Go + version: 1.0.0 +components: + - container: + endpoints: + - name: http + targetPort: 8080 + image: golang:latest + memoryLimit: 1024Mi + mountSources: true + sourceMapping: /project + name: runtime2 +commands: + - exec: + commandLine: GOCACHE=/project/.cache go build main.go + component: runtime2 + group: + isDefault: true + kind: build + workingDir: /project + id: build2 + - exec: + commandLine: ./main + component: runtime2 + group: + isDefault: true + kind: run + workingDir: /project + id: run2 +` + var mockDockerfile = ` FROM python:slim @@ -249,6 +356,16 @@ func (m MockGitUrl) DownloadInMemoryWithClient(params HTTPRequestParams, httpCli default: return []byte(mockDevfile), nil } + } else if m.GetToken() == "parent-devfile" { + if options.MockParent != nil && !options.MockParent.IsMainDevfileDownloaded { + options.MockParent.IsMainDevfileDownloaded = true + return []byte(mockDevfileWithParentRef), nil + } + + if options.MockParent != nil && !options.MockParent.IsParentDevfileDownloaded { + options.MockParent.IsParentDevfileDownloaded = true + return []byte(mockParentDevfile), nil + } } else if m.GetToken() == "" { // if no token is provided, assume normal operation return DownloadInMemory(params) From 10a737a8e8a1587d905ae9953cd3b48192e3d5e0 Mon Sep 17 00:00:00 2001 From: Maysun J Faisal Date: Tue, 5 Dec 2023 17:05:37 -0500 Subject: [PATCH 2/2] Add tests for the mock implementation Signed-off-by: Maysun J Faisal --- pkg/devfile/parser/parse_test.go | 31 +++++++++++++++++++++--- pkg/devfile/parser/util/utils_test.go | 34 +++++++++++++++++++++------ pkg/util/mock.go | 8 +++---- 3 files changed, 59 insertions(+), 14 deletions(-) diff --git a/pkg/devfile/parser/parse_test.go b/pkg/devfile/parser/parse_test.go index ab95c806..c778923f 100644 --- a/pkg/devfile/parser/parse_test.go +++ b/pkg/devfile/parser/parse_test.go @@ -4260,6 +4260,7 @@ func Test_parseFromURI_GitProviders(t *testing.T) { wantResources []string wantResourceContent []byte downloadGitResources bool + noMockData bool }{ { name: "private parent devfile", @@ -4281,6 +4282,26 @@ func Test_parseFromURI_GitProviders(t *testing.T) { wantResourceContent: []byte("private repo\ngit switched"), downloadGitResources: true, }, + { + name: "private parent devfile without mock data", + url: validUrl, + devfileUtilsClient: parserUtil.MockDevfileUtilsClient{ + DownloadOptions: util.MockDownloadOptions{ + MockFile: minimalDevfileContent, + }, + }, + token: validToken, + importReference: v1.ImportReference{ + ImportReferenceUnion: v1.ImportReferenceUnion{ + Uri: "https://github.com/private-url-devfile", + }, + }, + wantDevFile: minimalDevfile, + wantResources: []string{"resource.file"}, + wantResourceContent: []byte("private repo\ngit switched"), + downloadGitResources: true, + noMockData: true, + }, { name: "public parent devfile", url: validUrl, @@ -4444,9 +4465,13 @@ func Test_parseFromURI_GitProviders(t *testing.T) { t.Errorf("Unexpected err: %+v", err) } - tt.devfileUtilsClient.ParentURLAlias = tt.url - tt.devfileUtilsClient.GitTestToken = tt.token - tt.devfileUtilsClient.MockGitURL = util.MockGitUrl(*tt.gitUrl) + if tt.noMockData { + curDevfileContext.SetToken(tt.token) + } else { + tt.devfileUtilsClient.ParentURLAlias = tt.url + tt.devfileUtilsClient.GitTestToken = tt.token + tt.devfileUtilsClient.MockGitURL = util.MockGitUrl(*tt.gitUrl) + } got, err := parseFromURI(tt.importReference, curDevfileContext, &resolutionContextTree{}, resolverTools{downloadGitResources: tt.downloadGitResources, devfileUtilsClient: &tt.devfileUtilsClient}) diff --git a/pkg/devfile/parser/util/utils_test.go b/pkg/devfile/parser/util/utils_test.go index b4b00e33..b69f0f3b 100644 --- a/pkg/devfile/parser/util/utils_test.go +++ b/pkg/devfile/parser/util/utils_test.go @@ -48,12 +48,13 @@ func TestDownloadInMemoryClient(t *testing.T) { devfileUtilsClient := NewDevfileUtilsClient() tests := []struct { - name string - url string - token string - client DevfileUtils - want []byte - wantErr string + name string + url string + token string + client DevfileUtils + want []byte + wantParent []byte + wantErr string }{ { name: "Case 1: Input url is valid", @@ -112,6 +113,14 @@ func TestDownloadInMemoryClient(t *testing.T) { url: "https://raw.githubusercontent.com/maysunfaisal/OK/main/OK.txt", want: []byte{79, 75}, }, + { + name: "Case 10: Test devfile with private parent", + client: &MockDevfileUtilsClient{}, + url: "https://github.com/devfile/library/blob/main/devfile.yaml", + token: "parent-devfile", + want: []byte(util.MockDevfileWithParentRef), + wantParent: []byte(util.MockParentDevfile), + }, } for _, tt := range tests { @@ -120,10 +129,21 @@ func TestDownloadInMemoryClient(t *testing.T) { if (err != nil) != (tt.wantErr != "") { t.Errorf("Failed to download file with error: %s", err) } else if err == nil && !reflect.DeepEqual(data, tt.want) { - t.Errorf("Expected: %v, received: %v, difference at %v", tt.want, string(data[:]), pretty.Compare(tt.want, data)) + t.Errorf("Expected: %v, received: %v, difference at %v", string(tt.want), string(data[:]), pretty.Compare(tt.want, data)) } else if err != nil { assert.Regexp(t, tt.wantErr, err.Error(), "Error message should match") } + + if len(tt.wantParent) > 0 { + data, err := tt.client.DownloadInMemory(util.HTTPRequestParams{URL: tt.url, Token: tt.token}) + if (err != nil) != (tt.wantErr != "") { + t.Errorf("Failed to download file with error: %s", err) + } else if err == nil && !reflect.DeepEqual(data, tt.wantParent) { + t.Errorf("Expected: %v, received: %v, difference at %v", string(tt.wantParent), string(data[:]), pretty.Compare(tt.wantParent, data)) + } else if err != nil { + assert.Regexp(t, tt.wantErr, err.Error(), "Error message should match") + } + } }) } } diff --git a/pkg/util/mock.go b/pkg/util/mock.go index fb30ea61..7a0b7759 100644 --- a/pkg/util/mock.go +++ b/pkg/util/mock.go @@ -214,7 +214,7 @@ commands: isDefault: true ` -var mockDevfileWithParentRef = ` +var MockDevfileWithParentRef = ` schemaVersion: 2.2.0 metadata: displayName: Go Mock Runtime @@ -286,7 +286,7 @@ commands: isDefault: true ` -var mockParentDevfile = ` +var MockParentDevfile = ` schemaVersion: 2.2.0 metadata: displayName: Go Mock Parent @@ -359,12 +359,12 @@ func (m MockGitUrl) DownloadInMemoryWithClient(params HTTPRequestParams, httpCli } else if m.GetToken() == "parent-devfile" { if options.MockParent != nil && !options.MockParent.IsMainDevfileDownloaded { options.MockParent.IsMainDevfileDownloaded = true - return []byte(mockDevfileWithParentRef), nil + return []byte(MockDevfileWithParentRef), nil } if options.MockParent != nil && !options.MockParent.IsParentDevfileDownloaded { options.MockParent.IsParentDevfileDownloaded = true - return []byte(mockParentDevfile), nil + return []byte(MockParentDevfile), nil } } else if m.GetToken() == "" { // if no token is provided, assume normal operation