cmd/go/internal/modfetch: update for new gitrepo backend

Drop API-based access to Bitbucket, Gerrit, GitHub,
now that we can (again) talk directly to their Git repos.

Also drop general LatestAt query in favor of plain Latest
(at current time on default branch), which is all that we
needed and all that makes sense when talking directly
to Git servers.

For golang/go#24915 (still need hg, svn, bzr, fossil).
Fixes golang/go#23955 (no GitHub API calls, so no rate limits).
Fixes golang/go#24085 (no GitHub API calls, so no 403 responses).
Fixes golang/go#24076 (GitHub enterprise's go-get=1 pages are now OK).
Fixes golang/go#24102 (no more canonical GitHub import path checks).

Change-Id: I99fadb09a565398da3ec88f17691217aca7bf571
Reviewed-on: https://go-review.googlesource.com/107657
Run-TryBot: Russ Cox <[email protected]>
Reviewed-by: Bryan C. Mills <[email protected]>

Author: rsc

vendor/cmd/go/internal/modfetch/bitbucket/fetch.go

@@ -6,173 +6,17 @@ package bitbucket
 
 import (
 	"fmt"
-	"io"
-	"net/http"
-	"net/url"
 	"strings"
-	"time"
 
 	"cmd/go/internal/modfetch/codehost"
-	web "cmd/go/internal/web2"
+	"cmd/go/internal/modfetch/gitrepo"
 )
 
 func Lookup(path string) (codehost.Repo, error) {
 	f := strings.Split(path, "/")
 	if len(f) < 3 || f[0] != "bitbucket.org" {
 		return nil, fmt.Errorf("bitbucket repo must be bitbucket.org/org/project")
 	}
-
-	// Check for moved, renamed, or incorrect case in repository.
-	// Otherwise the API calls all appear to work.
-	// We need to do better here, but it's unclear exactly what.
-	var data struct {
-		FullName string `json:"full_name"`
-	}
-	err := web.Get("https://api.bitbucket.org/2.0/repositories/"+url.PathEscape(f[1])+"/"+url.PathEscape(f[2]), web.DecodeJSON(&data))
-	if err != nil {
-		return nil, err
-	}
-	myFullName := f[1] + "/" + f[2]
-	if myFullName != data.FullName {
-		why := "moved"
-		if strings.EqualFold(myFullName, data.FullName) {
-			why = "wrong case"
-		}
-		return nil, fmt.Errorf("module path of repo is bitbucket.org/%s, not %s (%s)", data.FullName, path, why)
-	}
-
-	return newRepo(f[1], f[2]), nil
-}
-
-func newRepo(owner, repository string) codehost.Repo {
-	return &repo{owner: owner, repo: repository}
-}
-
-type repo struct {
-	owner string
-	repo  string
-}
-
-func (r *repo) Root() string {
-	return "bitbucket.org/" + r.owner + "/" + r.repo
-}
-
-func (r *repo) Tags(prefix string) ([]string, error) {
-	var tags []string
-	u := "https://api.bitbucket.org/2.0/repositories/" + url.PathEscape(r.owner) + "/" + url.PathEscape(r.repo) + "/refs/tags"
-	var data struct {
-		Values []struct {
-			Name string `json:"name"`
-		} `json:"values"`
-	}
-	var hdr http.Header
-	err := web.Get(u, web.Header(&hdr), web.DecodeJSON(&data))
-	if err != nil {
-		return nil, err
-	}
-	for _, t := range data.Values {
-		if strings.HasPrefix(t.Name, prefix) {
-			tags = append(tags, t.Name)
-		}
-	}
-	return tags, nil
-}
-
-func (r *repo) LatestAt(t time.Time, branch string) (*codehost.RevInfo, error) {
-	u := "https://api.bitbucket.org/2.0/repositories/" + url.PathEscape(r.owner) + "/" + url.PathEscape(r.repo) + "/commits/" + url.QueryEscape(branch) + "?pagelen=10"
-	for u != "" {
-		var commits struct {
-			Values []struct {
-				Hash string `json:"hash"`
-				Date string `json:"date"`
-			} `json:"values"`
-			Next string `json:"next"`
-		}
-		err := web.Get(u, web.DecodeJSON(&commits))
-		if err != nil {
-			return nil, err
-		}
-		if len(commits.Values) == 0 {
-			return nil, fmt.Errorf("no commits")
-		}
-		for _, commit := range commits.Values {
-			d, err := time.Parse(time.RFC3339, commit.Date)
-			if err != nil {
-				return nil, err
-			}
-			if d.Before(t) {
-				info := &codehost.RevInfo{
-					Name:  commit.Hash,
-					Short: codehost.ShortenSHA1(commit.Hash),
-					Time:  d,
-				}
-				return info, nil
-			}
-		}
-		u = commits.Next
-	}
-	return nil, fmt.Errorf("no commits")
-}
-
-func (r *repo) Stat(rev string) (*codehost.RevInfo, error) {
-	var tag string
-	if !codehost.AllHex(rev) {
-		tag = rev
-	}
-	var commit struct {
-		Hash string `json:"hash"`
-		Type string `json:"type"`
-		Date string `json:"date"`
-	}
-	err := web.Get(
-		"https://api.bitbucket.org/2.0/repositories/"+url.PathEscape(r.owner)+"/"+url.PathEscape(r.repo)+"/commit/"+rev,
-		web.DecodeJSON(&commit),
-	)
-	if err != nil {
-		return nil, err
-	}
-	rev = commit.Hash
-	if rev == "" {
-		return nil, fmt.Errorf("no commits")
-	}
-	d, err := time.Parse(time.RFC3339, commit.Date)
-	if err != nil {
-		return nil, err
-	}
-	info := &codehost.RevInfo{
-		Name:    rev,
-		Short:   codehost.ShortenSHA1(rev),
-		Version: tag,
-		Time:    d,
-	}
-	return info, nil
-}
-
-func (r *repo) ReadFile(rev, file string, maxSize int64) ([]byte, error) {
-	// TODO: Use maxSize.
-	// TODO: I could not find an API endpoint for getting information about an
-	// individual file, and I do not know if the raw file download endpoint is
-	// a stable API.
-	var body []byte
-	err := web.Get(
-		"https://bitbucket.org/"+url.PathEscape(r.owner)+"/"+url.PathEscape(r.repo)+"/raw/"+url.PathEscape(rev)+"/"+url.PathEscape(file),
-		web.ReadAllBody(&body),
-	)
-	return body, err
-}
-
-func (r *repo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, actualSubdir string, err error) {
-	// TODO: Make web.Get copy to file for us, with limit.
-	var body io.ReadCloser
-	err = web.Get(
-		"https://bitbucket.org/"+url.PathEscape(r.owner)+"/"+url.PathEscape(r.repo)+"/get/"+url.PathEscape(rev)+".zip",
-		web.Body(&body),
-	)
-	if err != nil {
-		if body != nil {
-			body.Close()
-		}
-		return nil, "", err
-	}
-	return body, "", nil
+	path = f[0] + "/" + f[1] + "/" + f[2]
+	return gitrepo.Repo("https://"+path, path)
 }

vendor/cmd/go/internal/modfetch/codehost/codehost.go

@@ -29,11 +29,9 @@ type Repo interface {
 	// commit hash, branch, tag, and so on.
 	Stat(rev string) (*RevInfo, error)
 
-	// LatestAt returns the latest revision at the given time.
-	// If branch is non-empty, it restricts the query to revisions
-	// on the named branch. The meaning of "branch" depends
-	// on the underlying implementation.
-	LatestAt(t time.Time, branch string) (*RevInfo, error)
+	// Latest returns the latest revision on the default branch,
+	// whatever that means in the underlying implementation.
+	Latest() (*RevInfo, error)
 
 	// ReadFile reads the given file in the file tree corresponding to revision rev.
 	// It should refuse to read more than maxSize bytes.

vendor/cmd/go/internal/modfetch/coderepo.go

@@ -113,8 +113,8 @@ func (r *codeRepo) Stat(rev string) (*RevInfo, error) {
 	return r.convert(info)
 }
 
-func (r *codeRepo) LatestAt(t time.Time, branch string) (*RevInfo, error) {
-	info, err := r.code.LatestAt(t, branch)
+func (r *codeRepo) Latest() (*RevInfo, error) {
+	info, err := r.code.Latest()
 	if err != nil {
 		return nil, err
 	}

vendor/cmd/go/internal/modfetch/coderepo_test.go

@@ -10,6 +10,7 @@ import (
 	"cmd/go/internal/webtest"
 	"io"
 	"io/ioutil"
+	"log"
 	"os"
 	"reflect"
 	"strings"
@@ -21,6 +22,21 @@ func init() {
 	isTest = true
 }
 
+func TestMain(m *testing.M) {
+	os.Exit(testMain(m))
+}
+
+func testMain(m *testing.M) int {
+	dir, err := ioutil.TempDir("", "gitrepo-test-")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	codehost.WorkRoot = dir
+	return m.Run()
+}
+
 var codeRepoTests = []struct {
 	path     string
 	lookerr  string
@@ -119,12 +135,12 @@ var codeRepoTests = []struct {
 	{
 		path: "github.com/rsc/vgotest1/submod",
 		rev:  "v1.0.0",
-		err:  "404 Not Found", // TODO
+		err:  "unknown revision \"submod/v1.0.0\"",
 	},
 	{
 		path: "github.com/rsc/vgotest1/submod",
 		rev:  "v1.0.3",
-		err:  "404 Not Found", // TODO
+		err:  "unknown revision \"submod/v1.0.3\"",
 	},
 	{
 		path:    "github.com/rsc/vgotest1/submod",
@@ -228,16 +244,6 @@ var codeRepoTests = []struct {
 		time:    time.Date(2018, 2, 14, 0, 45, 20, 0, time.UTC),
 		gomod:   "module \"rsc.io/quote\"\n",
 	},
-	{
-		// redirect to bitbucket
-		path:    "example.net/vgotest",
-		rev:     "v1.0.0",
-		version: "v1.0.0",
-		name:    "9152736a7559a4b73ca45d36fe457cb617c3b207",
-		short:   "9152736a7559",
-		time:    time.Date(2017, 8, 22, 3, 10, 13, 0, time.UTC),
-		gomod:   "module \"example.net/vgotest\"\n",
-	},
 	{
 		// redirect to static hosting proxy
 		path:    "swtch.com/testmod",
@@ -308,13 +314,13 @@ var codeRepoTests = []struct {
 		gomod:   "module \"gopkg.in/yaml.v2\"\n\nrequire (\n\t\"gopkg.in/check.v1\" v0.0.0-20161208181325-20d25e280405\n)\n",
 	},
 	{
-		path:    "github.com/gobuffalo/buffalo",
-		rev:     "development",
-		version: "v0.0.0-20180406185414-59b4005674b6",
-		name:    "59b4005674b633728e2bfc3bb09cc204f7c2d6f5",
-		short:   "59b4005674b6",
-		time:    time.Date(2018, 4, 6, 18, 54, 14, 0, time.UTC),
-		gomod:   "//vgo 0.0.4\n\nmodule github.com/gobuffalo/buffalo\n",
+		path:    "vcs-test.golang.org/go/mod/gitrepo1",
+		rev:     "master",
+		version: "v0.0.0-20180417194322-ede458df7cd0",
+		name:    "ede458df7cd0fdca520df19a33158086a8a68e81",
+		short:   "ede458df7cd0",
+		time:    time.Date(2018, 4, 17, 19, 43, 22, 0, time.UTC),
+		gomod:   "//vgo 0.0.4\n\nmodule vcs-test.golang.org/go/mod/gitrepo1\n",
 	},
 }
 
@@ -528,63 +534,26 @@ func TestCodeRepoVersions(t *testing.T) {
 	}
 }
 
-var latestAtTests = []struct {
+var latestTests = []struct {
 	path    string
-	time    time.Time
-	branch  string
 	version string
 	err     string
 }{
 	{
-		path: "github.com/rsc/vgotest1",
-		time: time.Date(2018, 1, 20, 0, 0, 0, 0, time.UTC),
+		path: "github.com/rsc/empty",
 		err:  "no commits",
 	},
 	{
 		path:    "github.com/rsc/vgotest1",
-		time:    time.Date(2018, 2, 20, 0, 0, 0, 0, time.UTC),
 		version: "v0.0.0-20180219223237-a08abb797a67",
 	},
-	{
-		path:    "github.com/rsc/vgotest1",
-		time:    time.Date(2018, 2, 20, 0, 0, 0, 0, time.UTC),
-		branch:  "mybranch",
-		version: "v0.0.0-20180219231006-80d85c5d4d17",
-	},
 	{
 		path:    "swtch.com/testmod",
-		time:    time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC),
-		version: "v1.0.0",
-	},
-	{
-		path:    "swtch.com/testmod",
-		time:    time.Date(3000, 1, 1, 0, 0, 0, 0, time.UTC),
 		version: "v1.1.1",
 	},
-	{
-		path:   "swtch.com/testmod",
-		time:   time.Date(3000, 1, 1, 0, 0, 0, 0, time.UTC),
-		branch: "branch",
-		err:    "latest on branch not supported",
-	},
-	{
-		path:    "gopkg.in/check.v1",
-		time:    time.Date(2018, 2, 20, 15, 53, 33, 0, time.UTC),
-		version: "v0.0.0-20161208181325-20d25e280405",
-	},
-	{
-		path:    "gopkg.in/yaml.v2",
-		time:    time.Date(2018, 2, 20, 15, 53, 33, 0, time.UTC),
-		version: "v0.0.0-20180109114331-d670f9405373",
-	},
-	{
-		path:    "gopkg.in/russross/blackfriday.v2",
-		time:    time.Date(2018, 2, 20, 15, 53, 33, 0, time.UTC),
-		version: "v0.0.0-20180212083338-119f356b88f8",
-	},
 }
 
-func TestLatestAt(t *testing.T) {
+func TestLatest(t *testing.T) {
 	webtest.LoadOnce("testdata/webtest.txt")
 	webtest.Hook()
 	defer webtest.Unhook()
@@ -594,28 +563,25 @@ func TestLatestAt(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer os.RemoveAll(tmpdir)
-	for _, tt := range latestAtTests {
-		name := strings.Replace(tt.path, "/", "_", -1) + "/" + tt.time.Format("2006-01-02_15:04:05")
-		if tt.branch != "" {
-			name += "/" + tt.branch
-		}
+	for _, tt := range latestTests {
+		name := strings.Replace(tt.path, "/", "_", -1)
 		t.Run(name, func(t *testing.T) {
 			repo, err := Lookup(tt.path)
 			if err != nil {
 				t.Fatalf("Lookup(%q): %v", tt.path, err)
 			}
-			info, err := repo.LatestAt(tt.time, tt.branch)
+			info, err := repo.Latest()
 			if err != nil {
 				if tt.err != "" {
 					if err.Error() == tt.err {
 						return
 					}
-					t.Fatalf("LatestAt(%v, %q): %v, want %q", tt.time, tt.branch, err, tt.err)
+					t.Fatalf("Latest(): %v, want %q", err, tt.err)
 				}
-				t.Fatalf("LatestAt(%v, %q): %v", tt.time, tt.branch, err)
+				t.Fatalf("Latest(): %v", err)
 			}
 			if info.Version != tt.version {
-				t.Fatalf("LatestAt(%v, %q) = %v, want %v", tt.time, tt.branch, info.Version, tt.version)
+				t.Fatalf("Latest() = %v, want %v", info.Version, tt.version)
 			}
 		})
 	}