Wildcard ignores

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 NEW FEATURES:
 
+* Wildcard ignore support. (#1156)
 * Disable SourceManager lock by setting `DEPNOLOCK` environment variable.
 (#1206)
 

cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/final/Gopkg.toml

@@ -0,0 +1,5 @@
+ignored = ["github.com/golang/notexist/samples*"]
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/sdboyer/deptest"

cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/initial/Gopkg.toml

@@ -0,0 +1,5 @@
+ignored = ["github.com/golang/notexist/samples*"]
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/sdboyer/deptest"

cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/initial/main.go

@@ -0,0 +1,12 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	_ "github.com/sdboyer/deptest"
+)
+
+func main() {
+}

cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/initial/samples/samples.go

@@ -0,0 +1,7 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package samples
+
+import _ "github.com/sdboyer/deptestdos"

cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/initial/samples/subsamples/subsamples.go

@@ -0,0 +1,7 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package subsamples
+
+import _ "github.com/sdboyer/dep-test"

cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/testcase.json

@@ -0,0 +1,9 @@
+{
+  "commands": [
+    ["ensure"]
+  ],
+  "error-expected": "",
+  "vendor-final": [
+    "github.com/sdboyer/deptest"
+  ]
+}

docs/Gopkg.toml.md

@@ -36,6 +36,12 @@ You might also try [virtualgo](https://github.com/GetStream/vg), which installs
 ignored = ["github.com/user/project/badpkg"]
 ```
 
+Use wildcard character (*) to define a package prefix to be ignored. Use this
+to ignore any package and their subpackages.
+```toml
+ignored = ["github.com/user/project/badpkg*"]
+```
+
 **Use this for:** preventing a package and any of that package's unique
 dependencies from being installed.
 

internal/gps/hash.go

@@ -13,6 +13,8 @@ import (
 	"strings"
 )
 
+const wcIgnoreSuffix = "*"
+
 // string headers used to demarcate sections in hash input creation
 const (
 	hhConstraints = "-CONSTRAINTS-"
@@ -81,10 +83,24 @@ func (s *solver) writeHashingInputs(w io.Writer) {
 	writeString(hhIgnores)
 	ig := make([]string, 0, len(s.rd.ig))
 	for pkg := range s.rd.ig {
+		// Skip wildcard ignore. They are handled separately below.
+		if strings.HasSuffix(pkg, wcIgnoreSuffix) {
+			continue
+		}
+
 		if !strings.HasPrefix(pkg, s.rd.rpt.ImportRoot) || !isPathPrefixOrEqual(s.rd.rpt.ImportRoot, pkg) {
 			ig = append(ig, pkg)
 		}
 	}
+
+	// Add wildcard ignores to ignore list.
+	if s.rd.igpfx != nil {
+		s.rd.igpfx.Walk(func(s string, v interface{}) bool {
+			ig = append(ig, s+"*")
+			return false
+		})
+	}
+
 	sort.Strings(ig)
 
 	for _, igp := range ig {

internal/gps/hash_test.go

@@ -594,3 +594,94 @@ func diffHashingInputs(s Solver, wnt []string) string {
 	tw.Flush()
 	return buf.String()
 }
+
+func TestHashInputsIneffectualWildcardIgs(t *testing.T) {
+	fix := basicFixtures["shared dependency with overlapping constraints"]
+
+	rm := fix.rootmanifest().(simpleRootManifest).dup()
+
+	params := SolveParameters{
+		RootDir:         string(fix.ds[0].n),
+		RootPackageTree: fix.rootTree(),
+		Manifest:        rm,
+		ProjectAnalyzer: naiveAnalyzer{},
+		stdLibFn:        func(string) bool { return false },
+		mkBridgeFn:      overrideMkBridge,
+	}
+
+	cases := []struct {
+		name      string
+		ignoreMap map[string]bool
+		elems     []string
+	}{
+		{
+			name: "no wildcard ignores",
+			elems: []string{
+				hhConstraints,
+				"a",
+				"sv-1.0.0",
+				"b",
+				"sv-1.0.0",
+				hhImportsReqs,
+				"a",
+				"b",
+				hhIgnores,
+				hhOverrides,
+				hhAnalyzer,
+				"naive-analyzer",
+				"1",
+			},
+		},
+		{
+			name: "different wildcard ignores",
+			ignoreMap: map[string]bool{
+				"foobar*":    true,
+				"foobarbaz*": true,
+				"foozapbar*": true,
+			},
+			elems: []string{
+				hhConstraints,
+				"a",
+				"sv-1.0.0",
+				"b",
+				"sv-1.0.0",
+				hhImportsReqs,
+				"a",
+				"b",
+				hhIgnores,
+				"foobar*",
+				"foozapbar*",
+				hhOverrides,
+				hhAnalyzer,
+				"naive-analyzer",
+				"1",
+			},
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+
+			rm.ig = c.ignoreMap
+
+			params.Manifest = rm
+
+			s, err := Prepare(params, newdepspecSM(fix.ds, nil))
+			if err != nil {
+				t.Fatalf("Unexpected error while prepping solver: %s", err)
+			}
+
+			dig := s.HashInputs()
+			h := sha256.New()
+
+			for _, v := range c.elems {
+				h.Write([]byte(v))
+			}
+			correct := h.Sum(nil)
+
+			if !bytes.Equal(dig, correct) {
+				t.Errorf("Hashes are not equal. Inputs:\n%s", diffHashingInputs(s, c.elems))
+			}
+		})
+	}
+}

internal/gps/pkgtree/pkgtree.go

@@ -18,8 +18,13 @@ import (
 	"strconv"
 	"strings"
 	"unicode"
+
+	"github.com/armon/go-radix"
 )
 
+// wildcard ignore suffix
+const wcIgnoreSuffix = "*"
+
 // Package represents a Go package. It contains a subset of the information
 // go/build.Package does.
 type Package struct {
@@ -550,6 +555,9 @@ func (t PackageTree) ToReachMap(main, tests, backprop bool, ignore map[string]bo
 		ignore = make(map[string]bool)
 	}
 
+	// Radix tree for ignore prefixes.
+	xt := CreateIgnorePrefixTree(ignore)
+
 	// world's simplest adjacency list
 	workmap := make(map[string]wm)
 
@@ -572,6 +580,14 @@ func (t PackageTree) ToReachMap(main, tests, backprop bool, ignore map[string]bo
 			continue
 		}
 
+		if xt != nil {
+			// Skip ignored packages prefix matches
+			_, _, ok := xt.LongestPrefix(ip)
+			if ok {
+				continue
+			}
+		}
+
 		// TODO (kris-nova) Disable to get staticcheck passing
 		//imps = imps[:0]
 
@@ -1026,3 +1042,42 @@ func uniq(a []string) []string {
 	}
 	return a[:i]
 }
+
+// CreateIgnorePrefixTree takes a set of strings to be ignored and returns a
+// trie consisting of strings prefixed with wildcard ignore suffix (*).
+func CreateIgnorePrefixTree(ig map[string]bool) *radix.Tree {
+	var xt *radix.Tree
+
+	// Create a sorted list of all the ignores to have a proper order in
+	// ignores parsing.
+	sortedIgnores := make([]string, len(ig))
+	for k := range ig {
+		sortedIgnores = append(sortedIgnores, k)
+	}
+	sort.Strings(sortedIgnores)
+
+	for _, i := range sortedIgnores {
+		// Skip global ignore.
+		if i == "*" {
+			continue
+		}
+
+		// Check if it's a recursive ignore.
+		if strings.HasSuffix(i, wcIgnoreSuffix) {
+			// Create trie if it doesn't exists.
+			if xt == nil {
+				xt = radix.New()
+			}
+			// Check if it is ineffectual.
+			_, _, ok := xt.LongestPrefix(i)
+			if ok {
+				// Skip ineffectual wildcard ignore.
+				continue
+			}
+			// Create the ignore prefix and insert in the radix tree.
+			xt.Insert(i[:len(i)-len(wcIgnoreSuffix)], true)
+		}
+	}
+
+	return xt
+}

internal/gps/pkgtree/pkgtree_test.go

@@ -2038,3 +2038,92 @@ func getTestdataRootDir(t *testing.T) string {
 	}
 	return filepath.Join(cwd, "..", "_testdata")
 }
+
+func TestCreateIgnorePrefixTree(t *testing.T) {
+	cases := []struct {
+		name          string
+		ignoreMap     map[string]bool
+		wantInTree    []string
+		notWantInTree []string
+		wantNilTree   bool
+	}{
+		{
+			name:        "empty ignore",
+			wantNilTree: true,
+		},
+		{
+			name: "ignores without ignore suffix",
+			ignoreMap: map[string]bool{
+				"x/y/z":   true,
+				"*a/b/c":  true,
+				"gophers": true,
+			},
+			wantNilTree: true,
+		},
+		{
+			name: "ignores with ignore suffix",
+			ignoreMap: map[string]bool{
+				"x/y/z*":  true,
+				"a/b/c":   true,
+				"gophers": true,
+			},
+			wantInTree:    []string{"x/y/z"},
+			notWantInTree: []string{"a/b/c", "gophers"},
+		},
+		{
+			name:        "only skip global ignore",
+			ignoreMap:   map[string]bool{"*": true},
+			wantNilTree: true,
+		},
+		{
+			name: "global ignore with other strings",
+			ignoreMap: map[string]bool{
+				"*":        true,
+				"gophers*": true,
+				"x/y/z*":   true,
+				"a/b/c":    true,
+			},
+			wantInTree:    []string{"gophers", "x/y/z"},
+			notWantInTree: []string{"*", "a/b/c"},
+		},
+		{
+			name: "ineffectual ignore with wildcard",
+			ignoreMap: map[string]bool{
+				"a/b*":    true,
+				"a/b/c*":  true,
+				"a/b/x/y": true,
+				"a/c*":    true,
+			},
+			wantInTree:    []string{"a/b", "a/c"},
+			notWantInTree: []string{"a/b/c", "a/b/x/y"},
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			gotTree := CreateIgnorePrefixTree(c.ignoreMap)
+
+			if c.wantNilTree {
+				if gotTree != nil {
+					t.Fatalf("expected nil tree, got non-nil tree")
+				}
+			}
+
+			// Check if the wildcard suffix ignores are in the tree.
+			for _, p := range c.wantInTree {
+				_, has := gotTree.Get(p)
+				if !has {
+					t.Fatalf("expected %q to be in the tree", p)
+				}
+			}
+
+			// Check if non-wildcard suffix ignores are not in the tree.
+			for _, p := range c.notWantInTree {
+				_, has := gotTree.Get(p)
+				if has {
+					t.Fatalf("expected %q to not be in the tree", p)
+				}
+			}
+		})
+	}
+}

internal/gps/rootdata.go

@@ -20,6 +20,9 @@ type rootdata struct {
 	// Map of packages to ignore.
 	ig map[string]bool
 
+	// Radix tree of ignore prefixes.
+	igpfx *radix.Tree
+
 	// Map of packages to require.
 	req map[string]bool
 
@@ -202,3 +205,20 @@ func (rd rootdata) rootAtom() atomWithPackages {
 		pl: list,
 	}
 }
+
+// isIgnored checks if a given path is ignored, comparing with the list of
+// ignore packages and ignore prefixes.
+func (rd rootdata) isIgnored(path string) bool {
+	// Check if the path is present in ignore packages.
+	if rd.ig[path] {
+		return true
+	}
+
+	if rd.igpfx != nil {
+		// Check if the path matches any of the ignore prefixes.
+		_, _, ok := rd.igpfx.LongestPrefix(path)
+		return ok
+	}
+
+	return false
+}

internal/gps/rootdata_test.go

@@ -7,6 +7,8 @@ package gps
 import (
 	"reflect"
 	"testing"
+
+	"github.com/golang/dep/internal/gps/pkgtree"
 )
 
 func TestRootdataExternalImports(t *testing.T) {
@@ -222,3 +224,58 @@ func TestGetApplicableConstraints(t *testing.T) {
 		})
 	}
 }
+
+func TestIsIgnored(t *testing.T) {
+	cases := []struct {
+		name           string
+		ignorePkgs     map[string]bool
+		wantIgnored    []string
+		wantNotIgnored []string
+	}{
+		{
+			name: "no ignore",
+		},
+		{
+			name: "ignores without wildcard",
+			ignorePkgs: map[string]bool{
+				"a/b/c":   true,
+				"m/n":     true,
+				"gophers": true,
+			},
+			wantIgnored:    []string{"a/b/c", "m/n", "gophers"},
+			wantNotIgnored: []string{"somerandomstring"},
+		},
+		{
+			name: "ignores with wildcard",
+			ignorePkgs: map[string]bool{
+				"a/b/c*":    true,
+				"m/n*/o":    true,
+				"*x/y/z":    true,
+				"A/B*/C/D*": true,
+			},
+			wantIgnored:    []string{"a/b/c", "a/b/c/d", "a/b/c-d", "m/n*/o", "*x/y/z", "A/B*/C/D", "A/B*/C/D/E"},
+			wantNotIgnored: []string{"m/n/o", "m/n*", "x/y/z", "*x/y/z/a", "*x", "A/B", "A/B*/C"},
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			rd := rootdata{
+				ig:    c.ignorePkgs,
+				igpfx: pkgtree.CreateIgnorePrefixTree(c.ignorePkgs),
+			}
+
+			for _, p := range c.wantIgnored {
+				if !rd.isIgnored(p) {
+					t.Fatalf("expected %q to be ignored", p)
+				}
+			}
+
+			for _, p := range c.wantNotIgnored {
+				if rd.isIgnored(p) {
+					t.Fatalf("expected %q to be not ignored", p)
+				}
+			}
+		})
+	}
+}

internal/gps/solver.go

@@ -206,10 +206,13 @@ func (params SolveParameters) toRootdata() (rootdata, error) {
 		rd.ovr = make(ProjectConstraints)
 	}
 
+	// Create ignore prefix tree using the provided ignore packages
+	rd.igpfx = pkgtree.CreateIgnorePrefixTree(rd.ig)
+
 	if len(rd.ig) != 0 {
 		var both []string
 		for pkg := range params.Manifest.RequiredPackages() {
-			if rd.ig[pkg] {
+			if rd.isIgnored(pkg) {
 				both = append(both, pkg)
 			}
 		}
@@ -658,7 +661,7 @@ func (s *solver) getImportsAndConstraintsOf(a atomWithPackages) ([]string, []com
 	// explicitly listed in the atom
 	for _, pkg := range a.pl {
 		// Skip ignored packages
-		if s.rd.ig[pkg] {
+		if s.rd.isIgnored(pkg) {
 			continue
 		}
 

internal/gps/solver_inputs_test.go

@@ -114,6 +114,17 @@ func TestBadSolveOpts(t *testing.T) {
 	} else if !strings.Contains(err.Error(), "multiple packages given as both required and ignored:") {
 		t.Error("Prepare should have given error with multiple ignore/require conflict error, but gave:", err)
 	}
+
+	params.Manifest = simpleRootManifest{
+		ig:  map[string]bool{"foo*": true},
+		req: map[string]bool{"foo/bar": true},
+	}
+	_, err = Prepare(params, sm)
+	if err == nil {
+		t.Errorf("Should have errored on pkg both ignored (with wildcard) and required")
+	} else if !strings.Contains(err.Error(), "was given as both a required and ignored package") {
+		t.Error("Prepare should have given error with single ignore/require conflict error, but gave:", err)
+	}
 	params.Manifest = nil
 
 	params.ToChange = []ProjectRoot{"foo"}