dep, internal/gps: update SafeWriter.Write and gps.WriteDepTree

Signed-off-by: Ibrahim AshShohail <[email protected]>

Author: ibrasho

cmd/dep/ensure.go

@@ -252,7 +252,9 @@ func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project
 		if !ctx.Verbose {
 			logger = log.New(ioutil.Discard, "", 0)
 		}
-		return errors.WithMessage(sw.Write(p.AbsRoot, sm, true, logger), "grouped write of manifest, lock and vendor")
+
+		err = sw.Write(p.AbsRoot, sm, true, p.Manifest.PruneOptions, logger)
+		return errors.WithMessage(err, "grouped write of manifest, lock and vendor")
 	}
 
 	solution, err := solver.Solve()
@@ -273,7 +275,9 @@ func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project
 	if !ctx.Verbose {
 		logger = log.New(ioutil.Discard, "", 0)
 	}
-	return errors.Wrap(sw.Write(p.AbsRoot, sm, false, logger), "grouped write of manifest, lock and vendor")
+
+	err = sw.Write(p.AbsRoot, sm, false, p.Manifest.PruneOptions, logger)
+	return errors.Wrap(err, "grouped write of manifest, lock and vendor")
 }
 
 func (cmd *ensureCommand) runVendorOnly(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters) error {
@@ -306,7 +310,9 @@ func (cmd *ensureCommand) runVendorOnly(ctx *dep.Ctx, args []string, p *dep.Proj
 	if !ctx.Verbose {
 		logger = log.New(ioutil.Discard, "", 0)
 	}
-	return errors.WithMessage(sw.Write(p.AbsRoot, sm, true, logger), "grouped write of manifest, lock and vendor")
+
+	err = sw.Write(p.AbsRoot, sm, true, p.Manifest.PruneOptions, logger)
+	return errors.WithMessage(err, "grouped write of manifest, lock and vendor")
 }
 
 func (cmd *ensureCommand) runUpdate(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters) error {
@@ -401,7 +407,9 @@ func (cmd *ensureCommand) runUpdate(ctx *dep.Ctx, args []string, p *dep.Project,
 	if !ctx.Verbose {
 		logger = log.New(ioutil.Discard, "", 0)
 	}
-	return errors.Wrap(sw.Write(p.AbsRoot, sm, false, logger), "grouped write of manifest, lock and vendor")
+
+	err = sw.Write(p.AbsRoot, sm, false, p.Manifest.PruneOptions, logger)
+	return errors.Wrap(err, "grouped write of manifest, lock and vendor")
 }
 
 func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters) error {
@@ -654,7 +662,9 @@ func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm
 	if !ctx.Verbose {
 		logger = log.New(ioutil.Discard, "", 0)
 	}
-	if err := errors.Wrap(sw.Write(p.AbsRoot, sm, true, logger), "grouped write of manifest, lock and vendor"); err != nil {
+
+	err = sw.Write(p.AbsRoot, sm, true, p.Manifest.PruneOptions, logger)
+	if err := errors.Wrap(err, "grouped write of manifest, lock and vendor"); err != nil {
 		return err
 	}
 

cmd/dep/init.go

@@ -213,7 +213,8 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error {
 	if !ctx.Verbose {
 		logger = log.New(ioutil.Discard, "", 0)
 	}
-	if err := sw.Write(root, sm, !cmd.noExamples, logger); err != nil {
+
+	if err := sw.Write(root, sm, !cmd.noExamples, p.Manifest.PruneOptions, logger); err != nil {
 		return errors.Wrap(err, "safe write of manifest and lock")
 	}
 

internal/gps/result.go

@@ -5,10 +5,12 @@
 package gps
 
 import (
-	"fmt"
 	"log"
 	"os"
 	"path/filepath"
+	"sync"
+
+	"github.com/pkg/errors"
 )
 
 // A Solution is returned by a solver run. It is mostly just a Lock, with some
@@ -43,38 +45,65 @@ type solution struct {
 	solv Solver
 }
 
-// WriteDepTree takes a basedir and a Lock, and exports all the projects
-// listed in the lock to the appropriate target location within the basedir.
+// WriteDepTree takes a baseDir and a Lock, and exports all the projects
+// listed in the lock to the appropriate target location within baseDir.
 //
-// If the goal is to populate a vendor directory, basedir should be the absolute
+// If the goal is to populate a vendor directory, baseDir should be the absolute
 // path to that vendor directory, not its parent (a project root, typically).
 //
-// It requires a SourceManager to do the work, and takes a flag indicating
-// whether or not to strip vendor directories contained in the exported
-// dependencies.
-func WriteDepTree(basedir string, l Lock, sm SourceManager, sv bool, logger *log.Logger) error {
+// It requires a SourceManager to do the work, and takes a PruneOptions
+// indicating the pruning options required for the exported dependencies.
+func WriteDepTree(baseDir string, l Lock, sm SourceManager, prune PruneOptions, logger *log.Logger) error {
+	if baseDir == "" {
+		return errors.New("must provide a non-empty baseDir")
+	}
 	if l == nil {
-		return fmt.Errorf("must provide non-nil Lock to WriteDepTree")
+		return errors.New("must provide a non-nil Lock to WriteDepTree")
+	}
+	if sm == nil {
+		return errors.New("must provide a non-nil SourceManager to WriteDepTree")
 	}
 
-	err := os.MkdirAll(basedir, 0777)
-	if err != nil {
+	if err := os.MkdirAll(baseDir, 0777); err != nil {
 		return err
 	}
 
-	// TODO(sdboyer) parallelize
+	var wg sync.WaitGroup
+	errCh := make(chan error, len(l.Projects()))
+
 	for _, p := range l.Projects() {
-		to := filepath.FromSlash(filepath.Join(basedir, string(p.Ident().ProjectRoot)))
+		wg.Add(1)
+		go func(p LockedProject) {
+			to := filepath.FromSlash(filepath.Join(baseDir, string(p.Ident().ProjectRoot)))
 
-		logger.Printf("Writing out %s@%s", p.Ident().errString(), p.Version())
-		err = sm.ExportProject(p.Ident(), p.Version(), to)
-		if err != nil {
-			removeAll(basedir)
-			return fmt.Errorf("error while exporting %s@%s: %s", p.Ident().errString(), p.Version(), err)
-		}
-		if sv {
-			filepath.Walk(to, stripVendor)
+			logger.Printf("Writing out %s@%s", p.Ident().errString(), p.Version())
+
+			if err := sm.ExportProject(p.Ident(), p.Version(), to); err != nil {
+				removeAll(to)
+				errCh <- errors.Wrapf(err, "failed to export %s", p.Ident().ProjectRoot)
+			}
+
+			wg.Done()
+		}(p)
+	}
+
+	wg.Wait()
+	close(errCh)
+
+	if len(errCh) > 0 {
+		logger.Println("Failed to write dep tree. The following errors occurred:")
+		for err := range errCh {
+			logger.Println(" * ", err)
 		}
+		removeAll(baseDir)
+		// TODO(ibrasho) handle multiple errors
+		return <-errCh
+	}
+
+	if err := Prune(baseDir, prune, l, logger); err != nil {
+		logger.Println("Failed to prune dep tree.")
+		removeAll(baseDir)
+		return err
 	}
 
 	return nil

internal/gps/result_test.go

@@ -13,9 +13,10 @@ import (
 	"testing"
 )
 
-var discardLogger = log.New(ioutil.Discard, "", 0)
-
-var basicResult solution
+var (
+	basicResult   solution
+	discardLogger = log.New(ioutil.Discard, "", 0)
+)
 
 func pi(n string) ProjectIdentifier {
 	return ProjectIdentifier{
@@ -93,13 +94,12 @@ func testWriteDepTree(t *testing.T) {
 	}
 
 	// nil lock/result should err immediately
-	err = WriteDepTree(tmp, nil, sm, true, discardLogger)
-	if err == nil {
+
+	if err = WriteDepTree(tmp, nil, sm, PruneNestedVendorDirs, discardLogger); err == nil {
 		t.Errorf("Should error if nil lock is passed to WriteDepTree")
 	}
 
-	err = WriteDepTree(tmp, r, sm, true, discardLogger)
-	if err != nil {
+	if err = WriteDepTree(tmp, r, sm, PruneNestedVendorDirs, discardLogger); err != nil {
 		t.Errorf("Unexpected error while creating vendor tree: %s", err)
 	}
 
@@ -146,7 +146,7 @@ func BenchmarkCreateVendorTree(b *testing.B) {
 			// ease manual inspection
 			os.RemoveAll(exp)
 			b.StartTimer()
-			err = WriteDepTree(exp, r, sm, true, discardLogger)
+			err = WriteDepTree(exp, r, sm, PruneNestedVendorDirs, discardLogger)
 			b.StopTimer()
 			if err != nil {
 				b.Errorf("unexpected error after %v iterations: %s", i, err)

txn_writer.go

@@ -253,15 +253,15 @@ func (sw SafeWriter) validate(root string, sm gps.SourceManager) error {
 	return nil
 }
 
-// Write saves some combination of config yaml, lock, and a vendor tree.
+// Write saves some combination of a manifest, a lock, and a vendor tree.
 // root is the absolute path of root dir in which to write.
 // sm is only required if vendor is being written.
 //
 // It first writes to a temp dir, then moves them in place if and only if all the write
 // operations succeeded. It also does its best to roll back if any moves fail.
 // This mostly guarantees that dep cannot exit with a partial write that would
 // leave an undefined state on disk.
-func (sw *SafeWriter) Write(root string, sm gps.SourceManager, examples bool, logger *log.Logger) error {
+func (sw *SafeWriter) Write(root string, sm gps.SourceManager, examples bool, prune gps.PruneOptions, logger *log.Logger) error {
 	err := sw.validate(root, sm)
 	if err != nil {
 		return err
@@ -313,7 +313,10 @@ func (sw *SafeWriter) Write(root string, sm gps.SourceManager, examples bool, lo
 	}
 
 	if sw.writeVendor {
-		err = gps.WriteDepTree(filepath.Join(td, "vendor"), sw.lock, sm, true, logger)
+		// Ensure that gps.PruneNestedVendorDirs is toggled on.
+		prune |= gps.PruneNestedVendorDirs
+
+		err = gps.WriteDepTree(filepath.Join(td, "vendor"), sw.lock, sm, prune, logger)
 		if err != nil {
 			return errors.Wrap(err, "error while writing out vendor tree")
 		}

txn_writer_test.go

@@ -26,7 +26,7 @@ func TestSafeWriter_BadInput_MissingRoot(t *testing.T) {
 	defer pc.Release()
 
 	sw, _ := NewSafeWriter(nil, nil, nil, VendorOnChanged)
-	err := sw.Write("", pc.SourceManager, true, discardLogger)
+	err := sw.Write("", pc.SourceManager, true, PruneNestedVendorDirs, discardLogger)
 
 	if err == nil {
 		t.Fatal("should have errored without a root path, but did not")
@@ -44,7 +44,7 @@ func TestSafeWriter_BadInput_MissingSourceManager(t *testing.T) {
 	pc.Load()
 
 	sw, _ := NewSafeWriter(nil, nil, pc.Project.Lock, VendorAlways)
-	err := sw.Write(pc.Project.AbsRoot, nil, true, discardLogger)
+	err := sw.Write(pc.Project.AbsRoot, nil, true, PruneNestedVendorDirs, discardLogger)
 
 	if err == nil {
 		t.Fatal("should have errored without a source manager when forceVendor is true, but did not")
@@ -92,7 +92,7 @@ func TestSafeWriter_BadInput_NonexistentRoot(t *testing.T) {
 	sw, _ := NewSafeWriter(nil, nil, nil, VendorOnChanged)
 
 	missingroot := filepath.Join(pc.Project.AbsRoot, "nonexistent")
-	err := sw.Write(missingroot, pc.SourceManager, true, discardLogger)
+	err := sw.Write(missingroot, pc.SourceManager, true, PruneNestedVendorDirs, discardLogger)
 
 	if err == nil {
 		t.Fatal("should have errored with nonexistent dir for root path, but did not")
@@ -110,7 +110,7 @@ func TestSafeWriter_BadInput_RootIsFile(t *testing.T) {
 	sw, _ := NewSafeWriter(nil, nil, nil, VendorOnChanged)
 
 	fileroot := pc.CopyFile("fileroot", "txn_writer/badinput_fileroot")
-	err := sw.Write(fileroot, pc.SourceManager, true, discardLogger)
+	err := sw.Write(fileroot, pc.SourceManager, true, PruneNestedVendorDirs, discardLogger)
 
 	if err == nil {
 		t.Fatal("should have errored when root path is a file, but did not")
@@ -145,7 +145,7 @@ func TestSafeWriter_Manifest(t *testing.T) {
 	}
 
 	// Write changes
-	err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger)
+	err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, PruneNestedVendorDirs, discardLogger)
 	h.Must(errors.Wrap(err, "SafeWriter.Write failed"))
 
 	// Verify file system changes
@@ -190,7 +190,7 @@ func TestSafeWriter_ManifestAndUnmodifiedLock(t *testing.T) {
 	}
 
 	// Write changes
-	err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger)
+	err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, PruneNestedVendorDirs, discardLogger)
 	h.Must(errors.Wrap(err, "SafeWriter.Write failed"))
 
 	// Verify file system changes
@@ -235,7 +235,7 @@ func TestSafeWriter_ManifestAndUnmodifiedLockWithForceVendor(t *testing.T) {
 	}
 
 	// Write changes
-	err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger)
+	err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, PruneNestedVendorDirs, discardLogger)
 	h.Must(errors.Wrap(err, "SafeWriter.Write failed"))
 
 	// Verify file system changes
@@ -285,7 +285,7 @@ func TestSafeWriter_ModifiedLock(t *testing.T) {
 	}
 
 	// Write changes
-	err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger)
+	err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, PruneNestedVendorDirs, discardLogger)
 	h.Must(errors.Wrap(err, "SafeWriter.Write failed"))
 
 	// Verify file system changes
@@ -335,7 +335,7 @@ func TestSafeWriter_ModifiedLockSkipVendor(t *testing.T) {
 	}
 
 	// Write changes
-	err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger)
+	err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, PruneNestedVendorDirs, discardLogger)
 	h.Must(errors.Wrap(err, "SafeWriter.Write failed"))
 
 	// Verify file system changes
@@ -363,7 +363,7 @@ func TestSafeWriter_ForceVendorWhenVendorAlreadyExists(t *testing.T) {
 	pc.Load()
 
 	sw, _ := NewSafeWriter(nil, pc.Project.Lock, pc.Project.Lock, VendorAlways)
-	err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger)
+	err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, PruneNestedVendorDirs, discardLogger)
 	h.Must(errors.Wrap(err, "SafeWriter.Write failed"))
 
 	// Verify prepared actions
@@ -381,7 +381,7 @@ func TestSafeWriter_ForceVendorWhenVendorAlreadyExists(t *testing.T) {
 		t.Fatal("Expected the payload to contain the vendor directory ")
 	}
 
-	err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger)
+	err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, PruneNestedVendorDirs, discardLogger)
 	h.Must(errors.Wrap(err, "SafeWriter.Write failed"))
 
 	// Verify file system changes
@@ -431,7 +431,7 @@ func TestSafeWriter_NewLock(t *testing.T) {
 	}
 
 	// Write changes
-	err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger)
+	err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, PruneNestedVendorDirs, discardLogger)
 	h.Must(errors.Wrap(err, "SafeWriter.Write failed"))
 
 	// Verify file system changes
@@ -478,7 +478,7 @@ func TestSafeWriter_NewLockSkipVendor(t *testing.T) {
 	}
 
 	// Write changes
-	err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger)
+	err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, PruneNestedVendorDirs, discardLogger)
 	h.Must(errors.Wrap(err, "SafeWriter.Write failed"))
 
 	// Verify file system changes
@@ -571,7 +571,7 @@ func TestSafeWriter_VendorDotGitPreservedWithForceVendor(t *testing.T) {
 		t.Fatal("Expected the payload to contain the vendor directory")
 	}
 
-	err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger)
+	err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, PruneNestedVendorDirs, discardLogger)
 	h.Must(errors.Wrap(err, "SafeWriter.Write failed"))
 
 	// Verify file system changes