Merge pull request #1932 from sdboyer/dep-check

dep: Introduce dep check subcommand

Author: sdboyer

CHANGELOG.md

@@ -12,6 +12,7 @@ NEW FEATURES:
 * `dep ensure` now explains what changes to the code or Gopkg.toml have induced solving ([#1912](https://github.com/golang/dep/pull/1912)).
 * Hash digests of vendor contents are now stored in `Gopkg.lock`, and the contents of vendor are only rewritten on change or hash mismatch ([#1912](https://github.com/golang/dep/pull/1912)).
 * Added support for ppc64/ppc64le.
+* New subcommand `dep check` quickly reports if imports, Gopkg.toml, Gopkg.lock, and vendor are out of sync ([#1932](https://github.com/golang/dep/pull/1932)).
 
 BUG FIXES:
 
@@ -23,6 +24,7 @@ IMPROVEMENTS:
 * Reduce network access by trusting local source information and only pulling from upstream when necessary ([#1250](https://github.com/golang/dep/pull/1250)).
 * Update our dependency on Masterminds/semver to follow upstream again now that [Masterminds/semver#67](https://github.com/Masterminds/semver/pull/67) is merged([#1792](https://github.com/golang/dep/pull/1792)).
 * `inputs-digest` was removed from `Gopkg.lock` ([#1912](https://github.com/golang/dep/pull/1912)).
+* Hash digests of vendor contents are now stored in `Gopkg.lock`, and the contents of vendor are only rewritten on change or hash mismatch ([#1912](https://github.com/golang/dep/pull/1912)).
 * Don't exclude `Godeps` folder ([#1822](https://github.com/golang/dep/issues/1822)).
 * Add project-package relationship graph support in graphviz ([#1588](https://github.com/golang/dep/pull/1588)).
 

Makefile

@@ -18,8 +18,8 @@ licenseok:
 	go build -o licenseok ./hack/licenseok/main.go
 
 validate: build licenseok
+	./dep check
 	./hack/lint.bash
-	./hack/validate-vendor.bash
 	./hack/validate-licence.bash
 
 test: build

cmd/dep/check.go

@@ -0,0 +1,216 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/golang/dep"
+	"github.com/golang/dep/gps"
+	"github.com/golang/dep/gps/verify"
+	"github.com/pkg/errors"
+)
+
+const checkShortHelp = `Check if imports, Gopkg.toml, and Gopkg.lock are in sync`
+const checkLongHelp = `
+Check determines if your project is in a good state. If problems are found, it
+prints a description of each issue, then exits 1. Passing -q suppresses output.
+
+Flags control which specific checks will be run. By default, dep check verifies
+that Gopkg.lock is in sync with Gopkg.toml and the imports in your project's .go
+files, and that the vendor directory is in sync with Gopkg.lock. These checks
+can be disabled with -skip-lock and -skip-vendor, respectively.
+
+(See https://golang.github.io/dep/docs/ensure-mechanics.html#staying-in-sync for
+more information on what it means to be "in sync.")
+`
+
+type checkCommand struct {
+	quiet                bool
+	skiplock, skipvendor bool
+}
+
+func (cmd *checkCommand) Name() string { return "check" }
+func (cmd *checkCommand) Args() string {
+	return "[-q] [-skip-lock] [-skip-vendor]"
+}
+func (cmd *checkCommand) ShortHelp() string { return checkShortHelp }
+func (cmd *checkCommand) LongHelp() string  { return checkLongHelp }
+func (cmd *checkCommand) Hidden() bool      { return false }
+
+func (cmd *checkCommand) Register(fs *flag.FlagSet) {
+	fs.BoolVar(&cmd.skiplock, "skip-lock", false, "Skip checking that imports and Gopkg.toml are in sync with Gopkg.lock")
+	fs.BoolVar(&cmd.skipvendor, "skip-vendor", false, "Skip checking that vendor is in sync with Gopkg.lock")
+	fs.BoolVar(&cmd.quiet, "q", false, "Suppress non-error output")
+}
+
+func (cmd *checkCommand) Run(ctx *dep.Ctx, args []string) error {
+	logger := ctx.Out
+	if cmd.quiet {
+		logger = log.New(ioutil.Discard, "", 0)
+	}
+
+	p, err := ctx.LoadProject()
+	if err != nil {
+		return err
+	}
+
+	sm, err := ctx.SourceManager()
+	if err != nil {
+		return err
+	}
+
+	sm.UseDefaultSignalHandling()
+	defer sm.Release()
+
+	var fail bool
+	if !cmd.skiplock {
+		if p.Lock == nil {
+			return errors.New("Gopkg.lock does not exist, cannot check it against imports and Gopkg.toml")
+		}
+
+		lsat := verify.LockSatisfiesInputs(p.Lock, p.Manifest, p.RootPackageTree)
+		delta := verify.DiffLocks(p.Lock, p.ChangedLock)
+		sat, changed := lsat.Satisfied(), delta.Changed(verify.PruneOptsChanged|verify.HashVersionChanged)
+
+		if changed || !sat {
+			fail = true
+			logger.Println("# Gopkg.lock is out of sync:")
+			if !sat {
+				logger.Printf("%s\n", sprintLockUnsat(lsat))
+			}
+			if changed {
+				// Sort, for deterministic output.
+				var ordered []string
+				for pr := range delta.ProjectDeltas {
+					ordered = append(ordered, string(pr))
+				}
+				sort.Strings(ordered)
+
+				for _, pr := range ordered {
+					lpd := delta.ProjectDeltas[gps.ProjectRoot(pr)]
+					// Only two possible changes right now are prune opts
+					// changing or a missing hash digest (for old Gopkg.lock
+					// files)
+					if lpd.PruneOptsChanged() {
+						// Override what's on the lockdiff with the extra info we have;
+						// this lets us excise PruneNestedVendorDirs and get the real
+						// value from the input param in place.
+						old := lpd.PruneOptsBefore & ^gps.PruneNestedVendorDirs
+						new := lpd.PruneOptsAfter & ^gps.PruneNestedVendorDirs
+						logger.Printf("%s: prune options changed (%s -> %s)\n", pr, old, new)
+					}
+					if lpd.HashVersionWasZero() {
+						logger.Printf("%s: no hash digest in lock\n", pr)
+					}
+				}
+			}
+		}
+	}
+
+	if !cmd.skipvendor {
+		if p.Lock == nil {
+			return errors.New("Gopkg.lock does not exist, cannot check vendor against it")
+		}
+
+		statuses, err := p.VerifyVendor()
+		if err != nil {
+			return errors.Wrap(err, "error while verifying vendor")
+		}
+
+		if fail {
+			logger.Println()
+		}
+
+		var vendorfail bool
+		// One full pass through, to see if we need to print the header, and to
+		// create an array of names to sort for deterministic output.
+		var ordered []string
+		for path, status := range statuses {
+			ordered = append(ordered, path)
+			if status != verify.NoMismatch {
+				fail = true
+				if !vendorfail {
+					vendorfail = true
+					logger.Println("# vendor is out of sync:")
+				}
+			}
+		}
+		sort.Strings(ordered)
+
+		for _, pr := range ordered {
+			status := statuses[pr]
+			switch status {
+			case verify.NotInTree:
+				logger.Printf("%s: missing from vendor\n", pr)
+			case verify.NotInLock:
+				fi, err := os.Stat(filepath.Join(p.AbsRoot, "vendor", pr))
+				if err != nil {
+					return errors.Wrap(err, "could not stat file that VerifyVendor claimed existed")
+				}
+
+				if fi.IsDir() {
+					logger.Printf("%s: unused project\n", pr)
+				} else {
+					logger.Printf("%s: orphaned file\n", pr)
+				}
+			case verify.DigestMismatchInLock:
+				logger.Printf("%s: hash of vendored tree didn't match digest in Gopkg.lock\n", pr)
+			case verify.HashVersionMismatch:
+				// This will double-print if the hash version is zero, but
+				// that's a rare case that really only occurs before the first
+				// run with a version of dep >=0.5.0, so it's fine.
+				logger.Printf("%s: hash algorithm mismatch, want version %v\n", pr, verify.HashVersion)
+			}
+		}
+	}
+
+	if fail {
+		return silentfail{}
+	}
+	return nil
+}
+
+func sprintLockUnsat(lsat verify.LockSatisfaction) string {
+	var buf bytes.Buffer
+	sort.Strings(lsat.MissingImports)
+	for _, missing := range lsat.MissingImports {
+		fmt.Fprintf(&buf, "%s: missing from input-imports\n", missing)
+	}
+
+	sort.Strings(lsat.ExcessImports)
+	for _, excess := range lsat.ExcessImports {
+		fmt.Fprintf(&buf, "%s: in input-imports, but not imported\n", excess)
+	}
+
+	var ordered []string
+	for pr := range lsat.UnmetOverrides {
+		ordered = append(ordered, string(pr))
+	}
+	sort.Strings(ordered)
+	for _, pr := range ordered {
+		unmatched := lsat.UnmetOverrides[gps.ProjectRoot(pr)]
+		fmt.Fprintf(&buf, "%s@%s: not allowed by override %s\n", pr, unmatched.V, unmatched.C)
+	}
+
+	ordered = ordered[:0]
+	for pr := range lsat.UnmetConstraints {
+		ordered = append(ordered, string(pr))
+	}
+	sort.Strings(ordered)
+	for _, pr := range ordered {
+		unmatched := lsat.UnmetConstraints[gps.ProjectRoot(pr)]
+		fmt.Fprintf(&buf, "%s@%s: not allowed by constraint %s\n", pr, unmatched.V, unmatched.C)
+	}
+	return strings.TrimSpace(buf.String())
+}

cmd/dep/ensure.go

@@ -261,20 +261,7 @@ func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project
 		lsat := verify.LockSatisfiesInputs(p.Lock, p.Manifest, params.RootPackageTree)
 		if !lsat.Satisfied() {
 			if ctx.Verbose {
-				ctx.Out.Println("# Gopkg.lock is out of sync with Gopkg.toml and project code:")
-				for _, missing := range lsat.MissingImports {
-					ctx.Out.Printf("%s: missing from input-imports\n", missing)
-				}
-				for _, excess := range lsat.ExcessImports {
-					ctx.Out.Printf("%s: in input-imports, but isn't imported\n", excess)
-				}
-				for pr, unmatched := range lsat.UnmetOverrides {
-					ctx.Out.Printf("%s@%s: not allowed by override %s\n", pr, unmatched.V, unmatched.C)
-				}
-				for pr, unmatched := range lsat.UnmetConstraints {
-					ctx.Out.Printf("%s@%s: not allowed by constraint %s\n", pr, unmatched.V, unmatched.C)
-				}
-				ctx.Out.Println()
+				ctx.Out.Printf("# Gopkg.lock is out of sync with Gopkg.toml and project imports:\n%s\n\n", sprintLockUnsat(lsat))
 			}
 			solve = true
 		} else if cmd.noVendor {
@@ -329,7 +316,7 @@ func (cmd *ensureCommand) runVendorOnly(ctx *dep.Ctx, args []string, p *dep.Proj
 
 	// Pass the same lock as old and new so that the writer will observe no
 	// difference, and write out only ncessary vendor/ changes.
-	dw, err := dep.NewSafeWriter(nil, p.Lock, p.Lock, dep.VendorAlways, p.Manifest.PruneOptions)
+	dw, err := dep.NewSafeWriter(nil, p.Lock, p.Lock, dep.VendorAlways, p.Manifest.PruneOptions, nil)
 	//dw, err := dep.NewDeltaWriter(p.Lock, p.Lock, p.Manifest.PruneOptions, filepath.Join(p.AbsRoot, "vendor"), dep.VendorAlways)
 	if err != nil {
 		return err

cmd/dep/init.go

@@ -170,7 +170,7 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error {
 		ctx.Err.Printf("Old vendor backed up to %v", vendorbak)
 	}
 
-	sw, err := dep.NewSafeWriter(p.Manifest, nil, p.Lock, dep.VendorAlways, p.Manifest.PruneOptions)
+	sw, err := dep.NewSafeWriter(p.Manifest, nil, p.Lock, dep.VendorAlways, p.Manifest.PruneOptions, nil)
 	if err != nil {
 		return errors.Wrap(err, "init failed: unable to create a SafeWriter")
 	}

cmd/dep/main.go

@@ -39,6 +39,14 @@ type command interface {
 	Run(*dep.Ctx, []string) error
 }
 
+// Helper type so that commands can fail without generating any additional
+// ouptut.
+type silentfail struct{}
+
+func (silentfail) Error() string {
+	return ""
+}
+
 func main() {
 	p := &profile{}
 
@@ -139,7 +147,12 @@ func (c *Config) Run() int {
 			// Build flag set with global flags in there.
 			flags := flag.NewFlagSet(cmdName, flag.ContinueOnError)
 			flags.SetOutput(c.Stderr)
-			verbose := flags.Bool("v", false, "enable verbose logging")
+
+			var verbose bool
+			// No verbose for verify
+			if cmdName != "check" {
+				flags.BoolVar(&verbose, "v", false, "enable verbose logging")
+			}
 
 			// Register the subcommand flags in there, too.
 			cmd.Register(flags)
@@ -186,7 +199,7 @@ func (c *Config) Run() int {
 			ctx := &dep.Ctx{
 				Out:            outLogger,
 				Err:            errLogger,
-				Verbose:        *verbose,
+				Verbose:        verbose,
 				DisableLocking: getEnv(c.Env, "DEPNOLOCK") != "",
 				Cachedir:       cachedir,
 				CacheAge:       cacheAge,
@@ -197,7 +210,9 @@ func (c *Config) Run() int {
 
 			// Run the command with the post-flag-processing args.
 			if err := cmd.Run(ctx, flags.Args()); err != nil {
-				errLogger.Printf("%v\n", err)
+				if _, ok := err.(silentfail); !ok {
+					errLogger.Printf("%v\n", err)
+				}
 				return errorExitCode
 			}
 
@@ -222,6 +237,7 @@ func commandList() []command {
 		&ensureCommand{},
 		&pruneCommand{},
 		&versionCommand{},
+		&checkCommand{},
 	}
 }
 

cmd/dep/testdata/harness_tests/check/excess_inputs/final/Gopkg.lock

@@ -0,0 +1,12 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	_ "github.com/sdboyer/deptestdos"
+)
+
+func main() {
+}

cmd/dep/testdata/harness_tests/check/excess_inputs/final/Gopkg.toml

@@ -0,0 +1,3 @@
+# Gopkg.lock is out of sync:
+github.com/sdboyer/deptest: in input-imports, but not imported
+

cmd/dep/testdata/harness_tests/check/excess_inputs/initial/Gopkg.lock

@@ -0,0 +1,7 @@
+{
+  "commands": [
+    ["check"]
+  ],
+  "exit-code": 1,
+  "vendor-final": []
+}