updating tests

Signed-off-by: Michael Hoang <[email protected]>

Author: mike-hoang

README.md

@@ -12,8 +12,9 @@ The Devfile Parser library is a Golang module that:
 2. writes to the devfile.yaml with the updated data.
 3. generates Kubernetes objects for the various devfile resources.
 4. defines util functions for the devfile.
+5. downloads resources from a parent devfile if specified in the devfile.yaml
 
-## Private Repository Support
+## Private repository support
 
 Tokens are required to be set in the following cases:
 1. parsing a devfile from a private repository
@@ -24,19 +25,28 @@ Set the token for the repository:
 ```go
 parser.ParserArgs{
 	...
-	URL: <url-to-devfile-on-supported-git-provider>
+	// URL must point to a devfile.yaml
+	URL: <url-to-devfile-on-supported-git-provider-repo>/devfile.yaml
 	Token: <repo-personal-access-token>
 	...
 }
 ```
 Note: The url must also be set with a supported git provider repo url.
 
+Minimum token scope required:
+1. GitHub: Read access to code
+2. GitLab: Read repository
+3. Bitbucket: Read repository
+
+Note: To select token scopes for GitHub, a fine-grained token is required.
+
 For more information about personal access tokens:
 1. [GitHub docs](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token)
 2. [GitLab docs](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token)
 3. [Bitbucket docs](https://support.atlassian.com/bitbucket-cloud/docs/repository-access-tokens/)
 
 [1] Currently, this works under the assumption that the token can authenticate the devfile and the parent devfile; both devfiles are in the same repository.
+
 [2] In this scenario, the token will be used to authenticate the main devfile.
 
 ## Usage
@@ -199,6 +209,15 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g
    }
    ```
 
+9. When parsing a devfile that contains a parent reference, if the parent uri is a supported git provider repo url with the correct personal access token, all resources from the parent git repo excluding the parent devfile.yaml will be downloaded to the location of the devfile being parsed. **Note: The URL must point to a devfile.yaml**
+   ```yaml
+   schemaVersion: 2.2.0
+   ...
+   parent:
+      uri: <uri-to-parent-devfile>/devfile.yaml
+   ...
+   ```
+
 ## Projects using devfile/library
 
 The following projects are consuming this library as a Golang dependency

pkg/devfile/parser/context/context.go

@@ -73,14 +73,6 @@ func NewURLDevfileCtx(url string) DevfileCtx {
 	}
 }
 
-// NewPrivateURLDevfileCtx returns a new DevfileCtx type object
-func NewPrivateURLDevfileCtx(url string, token string) DevfileCtx {
-	return DevfileCtx{
-		url:   url,
-		token: token,
-	}
-}
-
 // NewByteContentDevfileCtx set devfile content from byte data and returns a new DevfileCtx type object and error
 func NewByteContentDevfileCtx(data []byte) (d DevfileCtx, err error) {
 	err = d.SetDevfileContentFromBytes(data)
@@ -166,6 +158,11 @@ func (d *DevfileCtx) GetToken() string {
 	return d.token
 }
 
+// SetToken sets the token for the devfile
+func (d *DevfileCtx) SetToken(token string) {
+	d.token = token
+}
+
 // SetAbsPath sets absolute file path for devfile
 func (d *DevfileCtx) SetAbsPath() (err error) {
 	// Set devfile absolute path

pkg/devfile/parser/context/context_test.go

@@ -88,16 +88,12 @@ func TestNewURLDevfileCtx(t *testing.T) {
 		token = "fake-token"
 		url   = "https://github.com/devfile/registry/blob/main/stacks/go/2.0.0/devfile.yaml"
 	)
-
-	{
-		d := NewPrivateURLDevfileCtx(url, token)
-		assert.Equal(t, "https://github.com/devfile/registry/blob/main/stacks/go/2.0.0/devfile.yaml", d.GetURL())
-		assert.Equal(t, "fake-token", d.GetToken())
-	}
 	{
 		d := NewURLDevfileCtx(url)
 		assert.Equal(t, "https://github.com/devfile/registry/blob/main/stacks/go/2.0.0/devfile.yaml", d.GetURL())
 		assert.Equal(t, "", d.GetToken())
+		d.SetToken(token)
+		assert.Equal(t, "fake-token", d.GetToken())
 	}
 }
 

pkg/devfile/parser/parse.go

@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/devfile/library/v2/pkg/git"
+	"github.com/hashicorp/go-multierror"
 	"io/ioutil"
 	"net/url"
 	"os"
@@ -49,34 +50,49 @@ import (
 
 // downloadGitRepoResources is exposed as a global variable for the purpose of running mock tests
 var downloadGitRepoResources = func(url string, destDir string, httpTimeout *int, token string) error {
+	var returnedErr error
+
 	gitUrl, err := git.NewGitUrlWithURL(url)
 	if err != nil {
 		return err
 	}
 
-	if gitUrl.IsGitProviderRepo() && gitUrl.IsFile {
-		stackDir, err := ioutil.TempDir(os.TempDir(), fmt.Sprintf("git-resources"))
+	if gitUrl.IsGitProviderRepo() {
+		if !gitUrl.IsFile || gitUrl.Revision == "" || !strings.Contains(gitUrl.Path, OutputDevfileYamlPath) {
+			return fmt.Errorf("error getting devfile from url: failed to retrieve %s", url)
+		}
+
+		stackDir, err := os.MkdirTemp("", fmt.Sprintf("git-resources"))
 		if err != nil {
 			return fmt.Errorf("failed to create dir: %s, error: %v", stackDir, err)
 		}
-		defer os.RemoveAll(stackDir)
+
+		defer func(path string) {
+			err := os.RemoveAll(path)
+			if err != nil {
+				returnedErr = multierror.Append(returnedErr, err)
+			}
+		}(stackDir)
 
 		if !gitUrl.IsPublic(httpTimeout) {
 			err = gitUrl.SetToken(token, httpTimeout)
 			if err != nil {
-				return err
+				returnedErr = multierror.Append(returnedErr, err)
+				return returnedErr
 			}
 		}
 
 		err = gitUrl.CloneGitRepo(stackDir)
 		if err != nil {
-			return err
+			returnedErr = multierror.Append(returnedErr, err)
+			return returnedErr
 		}
 
 		dir := path.Dir(path.Join(stackDir, gitUrl.Path))
 		err = git.CopyAllDirFiles(dir, destDir)
 		if err != nil {
-			return err
+			returnedErr = multierror.Append(returnedErr, err)
+			return returnedErr
 		}
 	}
 
@@ -163,15 +179,15 @@ func ParseDevfile(args ParserArgs) (d DevfileObj, err error) {
 	} else if args.Path != "" {
 		d.Ctx = devfileCtx.NewDevfileCtx(args.Path)
 	} else if args.URL != "" {
-		if args.Token != "" {
-			d.Ctx = devfileCtx.NewPrivateURLDevfileCtx(args.URL, args.Token)
-		} else {
-			d.Ctx = devfileCtx.NewURLDevfileCtx(args.URL)
-		}
+		d.Ctx = devfileCtx.NewURLDevfileCtx(args.URL)
 	} else {
 		return d, errors.Wrap(err, "the devfile source is not provided")
 	}
 
+	if args.Token != "" {
+		d.Ctx.SetToken(args.Token)
+	}
+
 	tool := resolverTools{
 		defaultNamespace: args.DefaultNamespace,
 		registryURLs:     args.RegistryURLs,
@@ -475,10 +491,9 @@ func parseFromURI(importReference v1.ImportReference, curDevfileCtx devfileCtx.D
 		}
 
 		token := curDevfileCtx.GetToken()
+		d.Ctx = devfileCtx.NewURLDevfileCtx(newUri)
 		if token != "" {
-			d.Ctx = devfileCtx.NewPrivateURLDevfileCtx(newUri, token)
-		} else {
-			d.Ctx = devfileCtx.NewURLDevfileCtx(newUri)
+			d.Ctx.SetToken(token)
 		}
 
 		destDir := path.Dir(curDevfileCtx.GetAbsPath())

pkg/devfile/parser/parse_test.go

@@ -18,6 +18,7 @@ package git
 import (
 	"fmt"
 	"net/url"
+	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
@@ -30,19 +31,19 @@ const (
 	BitbucketHost string = "bitbucket.org"
 )
 
-type Url struct {
+type GitUrl struct {
 	Protocol string // URL scheme
 	Host     string // URL domain name
 	Owner    string // name of the repo owner
 	Repo     string // name of the repo
 	Revision string // branch name, tag name, or commit id
 	Path     string // path to a directory or file in the repo
-	token    string // used for authenticating a private repo
+	token    string // authenticates private repo actions for parent devfiles
 	IsFile   bool   // defines if the URL points to a file in the repo
 }
 
 // NewGitUrlWithURL NewGitUrl creates a GitUrl from a string url
-func NewGitUrlWithURL(url string) (Url, error) {
+func NewGitUrlWithURL(url string) (GitUrl, error) {
 	gitUrl, err := ParseGitUrl(url)
 	if err != nil {
 		return gitUrl, err
@@ -52,8 +53,8 @@ func NewGitUrlWithURL(url string) (Url, error) {
 
 // ParseGitUrl extracts information from a support git url
 // Only supports git repositories hosted on GitHub, GitLab, and Bitbucket
-func ParseGitUrl(fullUrl string) (Url, error) {
-	var g Url
+func ParseGitUrl(fullUrl string) (GitUrl, error) {
+	var g GitUrl
 	err := ValidateURL(fullUrl)
 	if err != nil {
 		return g, err
@@ -81,7 +82,7 @@ func ParseGitUrl(fullUrl string) (Url, error) {
 	return g, err
 }
 
-func (g *Url) GetToken() string {
+func (g *GitUrl) GetToken() string {
 	return g.token
 }
 
@@ -106,7 +107,7 @@ var execute = func(baseDir string, cmd CommandType, args ...string) ([]byte, err
 	return []byte(""), fmt.Errorf(unsupportedCmdMsg, string(cmd))
 }
 
-func (g *Url) CloneGitRepo(destDir string) error {
+func (g *GitUrl) CloneGitRepo(destDir string) error {
 	exist := CheckPathExists(destDir)
 	if !exist {
 		return fmt.Errorf("failed to clone repo, destination directory: '%s' does not exists", destDir)
@@ -140,14 +141,18 @@ func (g *Url) CloneGitRepo(destDir string) error {
 	if g.Revision != "" {
 		_, err := execute(destDir, "git", "switch", "--detach", "origin/"+g.Revision)
 		if err != nil {
+			err = os.RemoveAll(destDir)
+			if err != nil {
+				return err
+			}
 			return fmt.Errorf("failed to switch repo to revision. repo dir: %v, revision: %v", destDir, g.Revision)
 		}
 	}
 
 	return nil
 }
 
-func (g *Url) parseGitHubUrl(url *url.URL) error {
+func (g *GitUrl) parseGitHubUrl(url *url.URL) error {
 	var splitUrl []string
 	var err error
 
@@ -209,7 +214,7 @@ func (g *Url) parseGitHubUrl(url *url.URL) error {
 	return err
 }
 
-func (g *Url) parseGitLabUrl(url *url.URL) error {
+func (g *GitUrl) parseGitLabUrl(url *url.URL) error {
 	var splitFile, splitOrg []string
 	var err error
 
@@ -257,7 +262,7 @@ func (g *Url) parseGitLabUrl(url *url.URL) error {
 	return err
 }
 
-func (g *Url) parseBitbucketUrl(url *url.URL) error {
+func (g *GitUrl) parseBitbucketUrl(url *url.URL) error {
 	var splitUrl []string
 	var err error
 
@@ -295,7 +300,7 @@ func (g *Url) parseBitbucketUrl(url *url.URL) error {
 
 // SetToken validates the token with a get request to the repo before setting the token
 // Defaults token to empty on failure.
-func (g *Url) SetToken(token string, httpTimeout *int) error {
+func (g *GitUrl) SetToken(token string, httpTimeout *int) error {
 	err := g.validateToken(HTTPRequestParams{Token: token, Timeout: httpTimeout})
 	if err != nil {
 		g.token = ""
@@ -307,7 +312,7 @@ func (g *Url) SetToken(token string, httpTimeout *int) error {
 
 // IsPublic checks if the GitUrl is public with a get request to the repo using an empty token
 // Returns true if the request succeeds
-func (g *Url) IsPublic(httpTimeout *int) bool {
+func (g *GitUrl) IsPublic(httpTimeout *int) bool {
 	err := g.validateToken(HTTPRequestParams{Token: "", Timeout: httpTimeout})
 	if err != nil {
 		return false
@@ -317,7 +322,7 @@ func (g *Url) IsPublic(httpTimeout *int) bool {
 
 // validateToken makes a http get request to the repo with the GitUrl token
 // Returns an error if the get request fails
-func (g *Url) validateToken(params HTTPRequestParams) error {
+func (g *GitUrl) validateToken(params HTTPRequestParams) error {
 	var apiUrl string
 
 	switch g.Host {
@@ -341,14 +346,14 @@ func (g *Url) validateToken(params HTTPRequestParams) error {
 }
 
 // GitRawFileAPI returns the endpoint for the git providers raw file
-func (g *Url) GitRawFileAPI() string {
+func (g *GitUrl) GitRawFileAPI() string {
 	var apiRawFile string
 
 	switch g.Host {
 	case GitHubHost, RawGitHubHost:
 		apiRawFile = fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", g.Owner, g.Repo, g.Revision, g.Path)
 	case GitLabHost:
-		apiRawFile = fmt.Sprintf("https://gitlab.com/api/v4/projects/%s%%2F%s/repository/files/%s/raw", g.Owner, g.Repo, g.Path)
+		apiRawFile = fmt.Sprintf("https://gitlab.com/api/v4/projects/%s%%2F%s/repository/files/%s/raw?ref=%s", g.Owner, g.Repo, g.Path, g.Revision)
 	case BitbucketHost:
 		apiRawFile = fmt.Sprintf("https://api.bitbucket.org/2.0/repositories/%s/%s/src/%s/%s", g.Owner, g.Repo, g.Revision, g.Path)
 	}
@@ -357,7 +362,7 @@ func (g *Url) GitRawFileAPI() string {
 }
 
 // IsGitProviderRepo checks if the url matches a repo from a supported git provider
-func (g *Url) IsGitProviderRepo() bool {
+func (g *GitUrl) IsGitProviderRepo() bool {
 	switch g.Host {
 	case GitHubHost, RawGitHubHost, GitLabHost, BitbucketHost:
 		return true

pkg/git/git.go

@@ -25,27 +25,41 @@ import (
 )
 
 func Test_ParseGitUrl(t *testing.T) {
+	invalidUrlError := "URL is invalid"
+	invalidUrlPathError := "url path to directory or file should contain*"
+	missingUserAndRepoError := "url path should contain <user>/<repo>*"
+
+	invalidGitHostError := "url host should be a valid GitHub, GitLab, or Bitbucket host*"
+	invalidGitHubPathError := "url path should contain <owner>/<repo>/<tree or blob>/<branch>/<path/to/file/or/directory>*"
+	invalidGitHubRawPathError := "raw url path should contain <owner>/<repo>/<branch>/<path/to/file>*"
+
+	invalidGitLabPathError := "url path to directory or file should contain <blob or tree or raw>/<branch>/<path/to/file/or/directory>*"
+	missingGitLabKeywordError := "url path should contain 'blob' or 'tree' or 'raw'*"
+
+	invalidBitbucketPathError := "url path should contain path to directory or file*"
+	missingBitbucketKeywordError := "url path should contain 'raw' or 'src'*"
+
 	tests := []struct {
 		name    string
 		url     string
-		wantUrl Url
+		wantUrl GitUrl
 		wantErr string
 	}{
 		{
 			name:    "should fail with empty url",
 			url:     "",
-			wantErr: "URL is invalid",
+			wantErr: invalidUrlError,
 		},
 		{
 			name:    "should fail with invalid git host",
 			url:     "https://google.ca/",
-			wantErr: "url host should be a valid GitHub, GitLab, or Bitbucket host*",
+			wantErr: invalidGitHostError,
 		},
 		// GitHub
 		{
 			name: "should parse GitHub repo with root path",
 			url:  "https://github.com/devfile/library",
-			wantUrl: Url{
+			wantUrl: GitUrl{
 				Protocol: "https",
 				Host:     "github.com",
 				Owner:    "devfile",
@@ -58,7 +72,7 @@ func Test_ParseGitUrl(t *testing.T) {
 		{
 			name: "should parse GitHub repo with root path and tag",
 			url:  "https://github.com/devfile/library/tree/v2.2.0",
-			wantUrl: Url{
+			wantUrl: GitUrl{
 				Protocol: "https",
 				Host:     "github.com",
 				Owner:    "devfile",
@@ -71,7 +85,7 @@ func Test_ParseGitUrl(t *testing.T) {
 		{
 			name: "should parse GitHub repo with root path and revision",
 			url:  "https://github.com/devfile/library/tree/0ce592a416fb185564516353891a45016ac7f671",
-			wantUrl: Url{
+			wantUrl: GitUrl{
 				Protocol: "https",
 				Host:     "github.com",
 				Owner:    "devfile",
@@ -84,12 +98,12 @@ func Test_ParseGitUrl(t *testing.T) {
 		{
 			name:    "should fail with only GitHub host",
 			url:     "https://github.com/",
-			wantErr: "url path should contain <user>/<repo>*",
+			wantErr: missingUserAndRepoError,
 		},
 		{
 			name: "should parse GitHub repo with file path",
 			url:  "https://github.com/devfile/library/blob/main/devfile.yaml",
-			wantUrl: Url{
+			wantUrl: GitUrl{
 				Protocol: "https",
 				Host:     "github.com",
 				Owner:    "devfile",
@@ -102,7 +116,7 @@ func Test_ParseGitUrl(t *testing.T) {
 		{
 			name: "should parse GitHub repo with raw file path",
 			url:  "https://raw.githubusercontent.com/devfile/library/main/devfile.yaml",
-			wantUrl: Url{
+			wantUrl: GitUrl{
 				Protocol: "https",
 				Host:     "raw.githubusercontent.com",
 				Owner:    "devfile",
@@ -115,38 +129,38 @@ func Test_ParseGitUrl(t *testing.T) {
 		{
 			name:    "should fail with missing GitHub repo",
 			url:     "https://github.com/devfile",
-			wantErr: "url path should contain <user>/<repo>*",
+			wantErr: missingUserAndRepoError,
 		},
 		{
 			name:    "should fail with missing GitHub blob",
 			url:     "https://github.com/devfile/library/main/devfile.yaml",
-			wantErr: "url path to directory or file should contain*",
+			wantErr: invalidUrlPathError,
 		},
 		{
 			name:    "should fail with missing GitHub tree",
 			url:     "https://github.com/devfile/library/main/tests/yamls",
-			wantErr: "url path to directory or file should contain*",
+			wantErr: invalidUrlPathError,
 		},
 		{
 			name:    "should fail with just GitHub tree",
 			url:     "https://github.com/devfile/library/tree",
-			wantErr: "url path should contain <owner>/<repo>/<tree or blob>/<branch>/<path/to/file/or/directory>*",
+			wantErr: invalidGitHubPathError,
 		},
 		{
 			name:    "should fail with just GitHub blob",
 			url:     "https://github.com/devfile/library/blob",
-			wantErr: "url path should contain <owner>/<repo>/<tree or blob>/<branch>/<path/to/file/or/directory>*",
+			wantErr: invalidGitHubPathError,
 		},
 		{
 			name:    "should fail with invalid GitHub raw file path",
 			url:     "https://raw.githubusercontent.com/devfile/library/devfile.yaml",
-			wantErr: "raw url path should contain <owner>/<repo>/<branch>/<path/to/file>*",
+			wantErr: invalidGitHubRawPathError,
 		},
 		// Gitlab
 		{
 			name: "should parse GitLab repo with root path",
 			url:  "https://gitlab.com/gitlab-org/gitlab-foss",
-			wantUrl: Url{
+			wantUrl: GitUrl{
 				Protocol: "https",
 				Host:     "gitlab.com",
 				Owner:    "gitlab-org",
@@ -159,12 +173,12 @@ func Test_ParseGitUrl(t *testing.T) {
 		{
 			name:    "should fail with only GitLab host",
 			url:     "https://gitlab.com/",
-			wantErr: "url path should contain <user>/<repo>*",
+			wantErr: missingUserAndRepoError,
 		},
 		{
 			name: "should parse GitLab repo with file path",
 			url:  "https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/README.md",
-			wantUrl: Url{
+			wantUrl: GitUrl{
 				Protocol: "https",
 				Host:     "gitlab.com",
 				Owner:    "gitlab-org",
@@ -177,23 +191,23 @@ func Test_ParseGitUrl(t *testing.T) {
 		{
 			name:    "should fail with missing GitLab repo",
 			url:     "https://gitlab.com/gitlab-org",
-			wantErr: "url path should contain <user>/<repo>*",
+			wantErr: missingUserAndRepoError,
 		},
 		{
 			name:    "should fail with missing GitLab keywords",
 			url:     "https://gitlab.com/gitlab-org/gitlab-foss/-/master/directory/README.md",
-			wantErr: "url path should contain 'blob' or 'tree' or 'raw'*",
+			wantErr: missingGitLabKeywordError,
 		},
 		{
 			name:    "should fail with missing GitLab file or directory path",
 			url:     "https://gitlab.com/gitlab-org/gitlab-foss/-/tree/master",
-			wantErr: "url path to directory or file should contain <blob or tree or raw>/<branch>/<path/to/file/or/directory>*",
+			wantErr: invalidGitLabPathError,
 		},
 		// Bitbucket
 		{
 			name: "should parse Bitbucket repo with root path",
 			url:  "https://bitbucket.org/fake-owner/fake-public-repo",
-			wantUrl: Url{
+			wantUrl: GitUrl{
 				Protocol: "https",
 				Host:     "bitbucket.org",
 				Owner:    "fake-owner",
@@ -206,12 +220,12 @@ func Test_ParseGitUrl(t *testing.T) {
 		{
 			name:    "should fail with only Bitbucket host",
 			url:     "https://bitbucket.org/",
-			wantErr: "url path should contain <user>/<repo>*",
+			wantErr: missingUserAndRepoError,
 		},
 		{
 			name: "should parse Bitbucket repo with file path",
 			url:  "https://bitbucket.org/fake-owner/fake-public-repo/src/main/README.md",
-			wantUrl: Url{
+			wantUrl: GitUrl{
 				Protocol: "https",
 				Host:     "bitbucket.org",
 				Owner:    "fake-owner",
@@ -224,7 +238,7 @@ func Test_ParseGitUrl(t *testing.T) {
 		{
 			name: "should parse Bitbucket file path with nested path",
 			url:  "https://bitbucket.org/fake-owner/fake-public-repo/src/main/directory/test.txt",
-			wantUrl: Url{
+			wantUrl: GitUrl{
 				Protocol: "https",
 				Host:     "bitbucket.org",
 				Owner:    "fake-owner",
@@ -237,7 +251,7 @@ func Test_ParseGitUrl(t *testing.T) {
 		{
 			name: "should parse Bitbucket repo with raw file path",
 			url:  "https://bitbucket.org/fake-owner/fake-public-repo/raw/main/README.md",
-			wantUrl: Url{
+			wantUrl: GitUrl{
 				Protocol: "https",
 				Host:     "bitbucket.org",
 				Owner:    "fake-owner",
@@ -250,17 +264,17 @@ func Test_ParseGitUrl(t *testing.T) {
 		{
 			name:    "should fail with missing Bitbucket repo",
 			url:     "https://bitbucket.org/fake-owner",
-			wantErr: "url path should contain <user>/<repo>*",
+			wantErr: missingUserAndRepoError,
 		},
 		{
 			name:    "should fail with invalid Bitbucket directory or file path",
 			url:     "https://bitbucket.org/fake-owner/fake-public-repo/main/README.md",
-			wantErr: "url path should contain path to directory or file*",
+			wantErr: invalidBitbucketPathError,
 		},
 		{
 			name:    "should fail with missing Bitbucket keywords",
 			url:     "https://bitbucket.org/fake-owner/fake-public-repo/main/test/README.md",
-			wantErr: "url path should contain 'raw' or 'src'*",
+			wantErr: missingBitbucketKeywordError,
 		},
 	}
 
@@ -281,12 +295,12 @@ func Test_ParseGitUrl(t *testing.T) {
 func Test_GetGitRawFileAPI(t *testing.T) {
 	tests := []struct {
 		name string
-		g    Url
+		g    GitUrl
 		want string
 	}{
 		{
 			name: "Github url",
-			g: Url{
+			g: GitUrl{
 				Protocol: "https",
 				Host:     "github.com",
 				Owner:    "devfile",
@@ -298,19 +312,19 @@ func Test_GetGitRawFileAPI(t *testing.T) {
 		},
 		{
 			name: "GitLab url",
-			g: Url{
+			g: GitUrl{
 				Protocol: "https",
 				Host:     "gitlab.com",
 				Owner:    "gitlab-org",
 				Repo:     "gitlab",
-				Revision: "master",
+				Revision: "v15.11.0-ee",
 				Path:     "README.md",
 			},
-			want: "https://gitlab.com/api/v4/projects/gitlab-org%2Fgitlab/repository/files/README.md/raw",
+			want: "https://gitlab.com/api/v4/projects/gitlab-org%2Fgitlab/repository/files/README.md/raw?ref=v15.11.0-ee",
 		},
 		{
 			name: "Bitbucket url",
-			g: Url{
+			g: GitUrl{
 				Protocol: "https",
 				Host:     "bitbucket.org",
 				Owner:    "owner",
@@ -322,7 +336,7 @@ func Test_GetGitRawFileAPI(t *testing.T) {
 		},
 		{
 			name: "Empty GitUrl",
-			g:    Url{},
+			g:    GitUrl{},
 			want: "",
 		},
 	}
@@ -338,7 +352,7 @@ func Test_GetGitRawFileAPI(t *testing.T) {
 }
 
 func Test_IsPublic(t *testing.T) {
-	publicGitUrl := Url{
+	publicGitUrl := GitUrl{
 		Protocol: "https",
 		Host:     "github.com",
 		Owner:    "devfile",
@@ -347,7 +361,7 @@ func Test_IsPublic(t *testing.T) {
 		token:    "fake-token",
 	}
 
-	privateGitUrl := Url{
+	privateGitUrl := GitUrl{
 		Protocol: "https",
 		Host:     "github.com",
 		Owner:    "not",
@@ -360,7 +374,7 @@ func Test_IsPublic(t *testing.T) {
 
 	tests := []struct {
 		name string
-		g    Url
+		g    GitUrl
 		want bool
 	}{
 		{
@@ -387,43 +401,16 @@ func Test_IsPublic(t *testing.T) {
 
 func Test_CloneGitRepo(t *testing.T) {
 	tempInvalidDir := t.TempDir()
-	tempDirGitHub := t.TempDir()
-	tempDirGitLab := t.TempDir()
-	tempDirBitbucket := t.TempDir()
 
-	invalidGitUrl := Url{
+	invalidGitUrl := GitUrl{
 		Protocol: "",
 		Host:     "",
 		Owner:    "nonexistent",
 		Repo:     "nonexistent",
 		Revision: "nonexistent",
 	}
 
-	validPublicGitHubUrl := Url{
-		Protocol: "https",
-		Host:     "github.com",
-		Owner:    "devfile",
-		Repo:     "library",
-		Revision: "main",
-	}
-
-	validPublicGitLabUrl := Url{
-		Protocol: "https",
-		Host:     "gitlab.com",
-		Owner:    "mike-hoang",
-		Repo:     "public-testing-repo",
-		Revision: "main",
-	}
-
-	validPublicBitbucketUrl := Url{
-		Protocol: "https",
-		Host:     "bitbucket.org",
-		Owner:    "mike-hoang",
-		Repo:     "public-testing-repo",
-		Revision: "master",
-	}
-
-	invalidPrivateGitHubRepo := Url{
+	invalidPrivateGitHubRepo := GitUrl{
 		Protocol: "https",
 		Host:     "github.com",
 		Owner:    "fake-owner",
@@ -438,7 +425,7 @@ func Test_CloneGitRepo(t *testing.T) {
 
 	tests := []struct {
 		name    string
-		gitUrl  Url
+		gitUrl  GitUrl
 		destDir string
 		wantErr string
 	}{
@@ -460,21 +447,6 @@ func Test_CloneGitRepo(t *testing.T) {
 			destDir: tempInvalidDir,
 			wantErr: privateRepoBadTokenErr,
 		},
-		{
-			name:    "should be able to clone valid public github url",
-			gitUrl:  validPublicGitHubUrl,
-			destDir: tempDirGitHub,
-		},
-		{
-			name:    "should be able to clone valid public gitlab url",
-			gitUrl:  validPublicGitLabUrl,
-			destDir: tempDirGitLab,
-		},
-		{
-			name:    "should be able to clone valid public bitbucket url",
-			gitUrl:  validPublicBitbucketUrl,
-			destDir: tempDirBitbucket,
-		},
 	}
 
 	for _, tt := range tests {

pkg/git/git_test.go

@@ -44,30 +44,43 @@ var mockExecute = func(baseDir string, cmd CommandType, args ...string) ([]byte,
 			u, _ := url.Parse(args[1])
 			password, hasPassword := u.User.Password()
 
+			resourceFile, err := os.Create(filepath.Clean(baseDir) + "/resource.file")
+			if err != nil {
+				return nil, fmt.Errorf("failed to create test resource: %v", err)
+			}
+
 			// private repository
 			if hasPassword {
 				switch password {
 				case "valid-token":
-					_, err := os.Create(filepath.Clean(baseDir) + "/private-repo-resource.txt")
+					_, err := resourceFile.WriteString("private repo\n")
 					if err != nil {
-						return nil, fmt.Errorf("failed to create test resource: %v", err)
+						return nil, fmt.Errorf("failed to write to test resource: %v", err)
 					}
-					return []byte("test"), nil
+					return []byte(""), nil
 				default:
 					return []byte(""), fmt.Errorf("not a valid token")
 				}
 			}
-			// public repository
-			_, err := os.Create(filepath.Clean(baseDir) + "/public-repo-resource.txt")
+
+			_, err = resourceFile.WriteString("public repo\n")
 			if err != nil {
-				return nil, fmt.Errorf("failed to create test resource: %v", err)
+				return nil, fmt.Errorf("failed to write to test resource: %v", err)
 			}
-			return []byte("test"), nil
+			return []byte(""), nil
 		}
 
 		if len(args) > 0 && args[0] == "switch" {
 			revision := strings.TrimPrefix(args[2], "origin/")
 			if revision != "invalid-revision" {
+				resourceFile, err := os.OpenFile(filepath.Clean(baseDir)+"/resource.file", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
+				if err != nil {
+					return nil, fmt.Errorf("failed to open test resource: %v", err)
+				}
+				_, err = resourceFile.WriteString("git switched")
+				if err != nil {
+					return nil, fmt.Errorf("failed to write to test resource: %v", err)
+				}
 				return []byte("git switched to revision"), nil
 			}
 			return []byte(""), fmt.Errorf("failed to switch revision")

pkg/git/mock.go

@@ -1100,7 +1100,7 @@ func DownloadInMemory(params HTTPRequestParams) ([]byte, error) {
 		ResponseHeaderTimeout: HTTPRequestResponseTimeout,
 	}, Timeout: HTTPRequestResponseTimeout}
 
-	var g git.Url
+	var g git.GitUrl
 	var err error
 
 	if IsGitProviderRepo(params.URL) {
@@ -1113,7 +1113,7 @@ func DownloadInMemory(params HTTPRequestParams) ([]byte, error) {
 	return downloadInMemoryWithClient(params, httpClient, g)
 }
 
-func downloadInMemoryWithClient(params HTTPRequestParams, httpClient HTTPClient, g git.Url) ([]byte, error) {
+func downloadInMemoryWithClient(params HTTPRequestParams, httpClient HTTPClient, g git.GitUrl) ([]byte, error) {
 	var url string
 	url = params.URL
 	req, err := http.NewRequest("GET", url, nil)
@@ -1128,6 +1128,7 @@ func downloadInMemoryWithClient(params HTTPRequestParams, httpClient HTTPClient,
 			return nil, err
 		}
 		if !g.IsPublic(params.Timeout) {
+			// check that the token is valid before adding to the header
 			err = g.SetToken(params.Token, params.Timeout)
 			if err != nil {
 				return nil, err