internal/gps: update prune functions to use in-memory filesystemState

This commit update prune functions to use filesystemState instead of
repeated walks over the filesyste.

strip_vendor functions still walk the filesystem and will be updated in
a later commit.

Signed-off-by: Ibrahim AshShohail <[email protected]>

Author: ibrasho

hack/lint.bash

@@ -9,4 +9,4 @@ set -e
 PKGS=$(go list ./... | grep -vF /vendor/)
 go vet $PKGS
 golint $PKGS
-megacheck -unused.exported -ignore "github.com/golang/dep/internal/test/test.go:U1000 github.com/golang/dep/internal/gps/prune.go:U1000" $PKGS
+megacheck -unused.exported -ignore "github.com/golang/dep/internal/test/test.go:U1000" $PKGS

internal/gps/prune.go

@@ -18,7 +18,7 @@ type PruneOptions uint8
 
 const (
 	// PruneNestedVendorDirs indicates if nested vendor directories should be pruned.
-	PruneNestedVendorDirs = 1 << iota
+	PruneNestedVendorDirs PruneOptions = 1 << iota
 	// PruneUnusedPackages indicates if unused Go packages should be pruned.
 	PruneUnusedPackages
 	// PruneNonGoFiles indicates if non-Go files should be pruned.
@@ -57,12 +57,13 @@ var (
 // on the PruneOptions passed.
 //
 // A Lock must be passed if PruneUnusedPackages is toggled on.
-func Prune(baseDir string, options PruneOptions, l Lock, logger *log.Logger) error {
+func Prune(baseDir string, l Lock, options PruneOptions, logger *log.Logger) error {
 	// TODO(ibrasho) allow passing specific options per project
+
 	for _, lp := range l.Projects() {
 		projectDir := filepath.Join(baseDir, string(lp.Ident().ProjectRoot))
-		err := PruneProject(projectDir, lp, options, logger)
-		if err != nil {
+
+		if err := PruneProject(projectDir, lp, options, logger); err != nil {
 			return err
 		}
 	}
@@ -73,28 +74,31 @@ func Prune(baseDir string, options PruneOptions, l Lock, logger *log.Logger) err
 // PruneProject remove excess files according to the options passed, from
 // the lp directory in baseDir.
 func PruneProject(baseDir string, lp LockedProject, options PruneOptions, logger *log.Logger) error {
-	projectDir := filepath.Join(baseDir, string(lp.Ident().ProjectRoot))
+	fs, err := deriveFilesystemState(baseDir)
+	if err != nil {
+		return errors.Wrap(err, "could not derive filesystem state")
+	}
 
 	if (options & PruneNestedVendorDirs) != 0 {
-		if err := pruneNestedVendorDirs(projectDir); err != nil {
+		if err := pruneNestedVendorDirs(baseDir); err != nil {
 			return err
 		}
 	}
 
 	if (options & PruneUnusedPackages) != 0 {
-		if err := pruneUnusedPackages(lp, projectDir, logger); err != nil {
+		if err := pruneUnusedPackages(lp, fs, logger); err != nil {
 			return errors.Wrap(err, "failed to prune unused packages")
 		}
 	}
 
 	if (options & PruneNonGoFiles) != 0 {
-		if err := pruneNonGoFiles(projectDir, logger); err != nil {
+		if err := pruneNonGoFiles(fs, logger); err != nil {
 			return errors.Wrap(err, "failed to prune non-Go files")
 		}
 	}
 
 	if (options & PruneGoTestFiles) != 0 {
-		if err := pruneGoTestFiles(projectDir, logger); err != nil {
+		if err := pruneGoTestFiles(fs, logger); err != nil {
 			return errors.Wrap(err, "failed to prune Go test files")
 		}
 	}
@@ -107,152 +111,141 @@ func pruneNestedVendorDirs(baseDir string) error {
 	return filepath.Walk(baseDir, stripVendor)
 }
 
-// pruneUnusedPackages deletes unimported packages found within baseDir.
+// pruneVendorDirs deletes all nested vendor directories within baseDir.
+// func pruneVendorDirs(fs filesystemState, logger *log.Logger) error {
+// 	toDelete := collectNestedVendorDirs(fs)
+
+// 	for _, path := range toDelete {
+// 		logger.Printf("  * %s", path)
+
+// 		if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
+// 			return err
+// 		}
+// 	}
+
+// 	return nil
+// }
+
+// func collectNestedVendorDirs(fs filesystemState) []string {
+// 	toDelete := make([]string, 0, len(fs.dirs)/4)
+
+// 	for _, dir := range fs.dirs {
+// 		if filepath.Base(dir)
+// 		toDelete = append(toDelete, dir)
+// 	}
+
+// 	for _, link := range fs.links {
+// 		toDelete = append(toDelete)
+// 	}
+
+// 	return files
+// }
+
+// pruneUnusedPackages deletes unimported packages found in fs.
 // Determining whether packages are imported or not is based on the passed LockedProject.
-func pruneUnusedPackages(lp LockedProject, projectDir string, logger *log.Logger) error {
+func pruneUnusedPackages(lp LockedProject, fs filesystemState, logger *log.Logger) error {
 	pr := string(lp.Ident().ProjectRoot)
-	logger.Printf("Calculating unused packages in %s to prune.\n", pr)
+	unusedPackages := calculateUnusedPackages(lp, fs)
 
-	unusedPackages, err := calculateUnusedPackages(lp, projectDir)
-	if err != nil {
-		return errors.Wrapf(err, "could not calculate unused packages in %s", pr)
-	}
+	logger.Printf("Found %d unused packages in %s:\n", len(unusedPackages), pr)
 
-	logger.Printf("Found the following unused packages in %s:\n", pr)
 	for pkg := range unusedPackages {
 		logger.Printf("  * %s\n", filepath.Join(pr, pkg))
 	}
 
-	unusedPackagesFiles, err := collectUnusedPackagesFiles(projectDir, unusedPackages)
-	if err != nil {
-		return errors.Wrapf(err, "could not collect unused packages' files in %s", pr)
-	}
+	toDelete := collectUnusedPackagesFiles(fs, unusedPackages)
+
+	logger.Printf("Deleting %d files in unused packages:", len(toDelete))
 
-	if err := deleteFiles(unusedPackagesFiles); err != nil {
-		return errors.Wrapf(err, "")
+	for _, path := range toDelete {
+		logger.Printf("  * %s\n", path)
+
+		if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
+			return err
+		}
 	}
 
 	return nil
 }
 
 // calculateUnusedPackages generates a list of unused packages in lp.
-func calculateUnusedPackages(lp LockedProject, projectDir string) (map[string]struct{}, error) {
-	// TODO(ibrasho): optimize this...
+func calculateUnusedPackages(lp LockedProject, fs filesystemState) map[string]struct{} {
 	unused := make(map[string]struct{})
 	imported := make(map[string]struct{})
+
 	for _, pkg := range lp.Packages() {
 		imported[pkg] = struct{}{}
 	}
 
-	err := filepath.Walk(projectDir, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-
-		// Ignore anything that's not a directory.
-		if !info.IsDir() {
-			return nil
-		}
+	// Add the root package if it's not imported.
+	if _, ok := imported["."]; !ok {
+		unused["."] = struct{}{}
+	}
 
-		pkg, err := filepath.Rel(projectDir, path)
-		if err != nil {
-			return errors.Wrap(err, "unexpected error while calculating unused packages")
-		}
+	for _, dirPath := range fs.dirs {
+		pkg := filepath.ToSlash(dirPath)
 
-		pkg = filepath.ToSlash(pkg)
 		if _, ok := imported[pkg]; !ok {
 			unused[pkg] = struct{}{}
 		}
+	}
 
-		return nil
-	})
-
-	return unused, err
+	return unused
 }
 
-// collectUnusedPackagesFiles returns a slice of all files in the unused packages in projectDir.
-func collectUnusedPackagesFiles(projectDir string, unusedPackages map[string]struct{}) ([]string, error) {
+// collectUnusedPackagesFiles returns a slice of all files in the unused packages based on fs.
+func collectUnusedPackagesFiles(fs filesystemState, unusedPackages map[string]struct{}) []string {
 	// TODO(ibrasho): is this useful?
 	files := make([]string, 0, len(unusedPackages))
 
-	err := filepath.Walk(projectDir, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
+	for _, path := range fs.files {
+		// Keep perserved files.
+		if isPreservedFile(filepath.Base(path)) {
+			continue
 		}
 
-		// Ignore directories.
-		if info.IsDir() {
-			return nil
-		}
+		pkg := filepath.ToSlash(filepath.Dir(path))
 
-		// Ignore perserved files.
-		if isPreservedFile(info.Name()) {
-			return nil
-		}
-
-		pkg, err := filepath.Rel(projectDir, filepath.Dir(path))
-		if err != nil {
-			return errors.Wrap(err, "unexpected error while calculating unused packages")
-		}
-
-		pkg = filepath.ToSlash(pkg)
 		if _, ok := unusedPackages[pkg]; ok {
-			files = append(files, path)
+			files = append(files, filepath.Join(fs.root, path))
 		}
+	}
 
-		return nil
-	})
-
-	return files, err
+	return files
 }
 
-// pruneNonGoFiles delete all non-Go files existing within baseDir.
+// pruneNonGoFiles delete all non-Go files existing in fs.
 // Files with names that are prefixed by any entry in preservedNonGoFiles
 // are not deleted.
-func pruneNonGoFiles(baseDir string, logger *log.Logger) error {
-	files, err := collectNonGoFiles(baseDir, logger)
-	if err != nil {
-		return errors.Wrap(err, "could not collect non-Go files")
-	}
-
-	if err := deleteFiles(files); err != nil {
-		return errors.Wrap(err, "could not prune Go test files")
-	}
-
-	return nil
-}
-
-// collectNonGoFiles returns a slice containing all non-Go files in baseDir.
-// Files meeting the checks in isPreservedFile are not returned.
-func collectNonGoFiles(baseDir string, logger *log.Logger) ([]string, error) {
-	files := make([]string, 0)
-
-	err := filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
+func pruneNonGoFiles(fs filesystemState, logger *log.Logger) error {
+	// TODO(ibrasho) detemine a sane capacity
+	toDelete := make([]string, 0, len(fs.files)/4)
+
+	for _, path := range fs.files {
+		// Ignore Go files.
+		if strings.HasSuffix(path, ".go") {
+			continue
 		}
 
-		// Ignore directories.
-		if info.IsDir() {
-			return nil
+		// Ignore perserved files.
+		if isPreservedFile(filepath.Base(path)) {
+			continue
 		}
 
-		// Ignore all Go files.
-		if strings.HasSuffix(info.Name(), ".go") {
-			return nil
-		}
+		toDelete = append(toDelete, filepath.Join(fs.root, path))
+	}
 
-		// Ignore perserved files.
-		if isPreservedFile(info.Name()) {
-			return nil
-		}
+	logger.Printf("Deleting %d non-Go files:\n", len(toDelete))
 
-		files = append(files, path)
+	for _, path := range toDelete {
+		logger.Printf("  * %s\n", path)
 
-		return nil
-	})
+		if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
+			return err
+		}
+	}
 
-	return files, err
+	return nil
 }
 
 // isPreservedFile checks if the file name indicates that the file should be
@@ -275,51 +268,26 @@ func isPreservedFile(name string) bool {
 	return false
 }
 
-// pruneGoTestFiles deletes all Go test files (*_test.go) within baseDir.
-func pruneGoTestFiles(baseDir string, logger *log.Logger) error {
-	files, err := collectGoTestFiles(baseDir)
-	if err != nil {
-		return errors.Wrap(err, "could not collect Go test files")
-	}
-
-	if err := deleteFiles(files); err != nil {
-		return errors.Wrap(err, "could not prune Go test files")
-	}
-
-	return nil
-}
-
-// collectGoTestFiles returns a slice contains all Go test files (any files
-// prefixed with _test.go) in baseDir.
-func collectGoTestFiles(baseDir string) ([]string, error) {
-	files := make([]string, 0)
+// pruneGoTestFiles deletes all Go test files (*_test.go) in fs.
+func pruneGoTestFiles(fs filesystemState, logger *log.Logger) error {
+	// TODO(ibrasho) detemine a sane capacity
+	toDelete := make([]string, 0, len(fs.files)/2)
 
-	err := filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-
-		// Ignore directories.
-		if info.IsDir() {
-			return nil
-		}
-
-		// Ignore any files that is not a Go test file.
-		if strings.HasSuffix(info.Name(), "_test.go") {
-			files = append(files, path)
+	for _, path := range fs.files {
+		if strings.HasSuffix(path, "_test.go") {
+			toDelete = append(toDelete, filepath.Join(fs.root, path))
 		}
+	}
 
-		return nil
-	})
+	logger.Printf("Deleting %d Go test files:\n", len(toDelete))
 
-	return files, err
-}
+	for _, path := range toDelete {
+		logger.Printf("  * %s\n", path)
 
-func deleteFiles(paths []string) error {
-	for _, path := range paths {
-		if err := os.Remove(path); err != nil {
+		if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
 			return err
 		}
 	}
+
 	return nil
 }