Prefetch and log output during InitializeRootManifestAndLock

Author: xmattstrongx

cmd/dep/root_analyzer.go

@@ -5,15 +5,22 @@
 package main
 
 import (
+	"context"
 	"io/ioutil"
 	"log"
+	"strings"
+
+	"golang.org/x/sync/errgroup"
 
 	"github.com/golang/dep"
 	fb "github.com/golang/dep/internal/feedback"
 	"github.com/golang/dep/internal/gps"
+	"github.com/golang/dep/internal/gps/paths"
 	"github.com/golang/dep/internal/importers"
 )
 
+const concurrency = 4
+
 // rootAnalyzer supplies manifest/lock data from both dep and external tool's
 // configuration files.
 // * When used on the root project, it imports only from external tools.
@@ -45,6 +52,9 @@ func (a *rootAnalyzer) InitializeRootManifestAndLock(dir string, pr gps.ProjectR
 
 	if rootM == nil {
 		rootM = dep.NewManifest()
+		if err := a.cacheDeps(pr); err != nil {
+			return nil, nil, err
+		}
 	}
 	if rootL == nil {
 		rootL = &dep.Lock{}
@@ -53,6 +63,65 @@ func (a *rootAnalyzer) InitializeRootManifestAndLock(dir string, pr gps.ProjectR
 	return
 }
 
+func (a *rootAnalyzer) cacheDeps(pr gps.ProjectRoot) error {
+	deps := make(map[gps.ProjectRoot]bool)
+	logger := a.ctx.Err
+	g, ctx := errgroup.WithContext(context.TODO())
+	sem := make(chan struct{}, concurrency)
+
+	syncDep := func(pr gps.ProjectRoot, sm gps.SourceManager) error {
+		if err := sm.SyncSourceFor(gps.ProjectIdentifier{ProjectRoot: pr}); err != nil {
+			logger.Printf("Unable to cache %s - %s", pr, err)
+			return err
+		}
+		logger.Printf("Cached %s", pr)
+		return nil
+	}
+
+	for ip := range a.directDeps {
+		logger.Printf("Package %q, analyzing...", ip)
+		if paths.IsStandardImportPath(ip) {
+			continue
+		}
+		if hasImportPathPrefix(ip, string(pr)) {
+			continue
+		}
+
+		pr, err := a.sm.DeduceProjectRoot(ip)
+		if err != nil {
+			return err
+		}
+
+		if _, ok := deps[pr]; ok {
+			continue
+		}
+
+		g.Go(func() error {
+			select {
+			case sem <- struct{}{}:
+				defer func() { <-sem }()
+			case <-ctx.Done():
+				return ctx.Err()
+			}
+			err := syncDep(pr, a.sm)
+			return err
+		})
+
+		deps[pr] = true
+	}
+	if err := g.Wait(); err == nil {
+		logger.Printf("Successfully cached all deps.")
+	}
+	return nil
+}
+
+func hasImportPathPrefix(s, prefix string) bool {
+	if s == prefix {
+		return true
+	}
+	return strings.HasPrefix(s, prefix+"/")
+}
+
 func (a *rootAnalyzer) importManifestAndLock(dir string, pr gps.ProjectRoot, suppressLogs bool) (*dep.Manifest, *dep.Lock, error) {
 	logger := a.ctx.Err
 	if suppressLogs {