Skip to content
This repository was archived by the owner on Sep 9, 2020. It is now read-only.
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 8344181

Browse files
ibrashoibrasho
committedAug 8, 2017
internal/gps: implement Prune and the related functions
Signed-off-by: Ibrahim AshShohail <[email protected]>
1 parent a702f59 commit 8344181

File tree

4 files changed

+366
-48
lines changed

4 files changed

+366
-48
lines changed
 

‎internal/gps/prune.go

+288
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,18 @@
44

55
package gps
66

7+
import (
8+
"fmt"
9+
"io/ioutil"
10+
"log"
11+
"os"
12+
"path/filepath"
13+
"sort"
14+
"strings"
15+
16+
"github.com/pkg/errors"
17+
)
18+
719
// PruneOptions represents the pruning options used to write the dependecy tree.
820
type PruneOptions uint8
921

@@ -25,3 +37,279 @@ var (
2537
"COPYING",
2638
}
2739
)
40+
41+
func Prune(baseDir string, options PruneOptions, l Lock, logger *log.Logger) error {
42+
if (options & PruneNestedVendorDirs) != 0 {
43+
if err := pruneNestedVendorDirs(baseDir); err != nil {
44+
return err
45+
}
46+
}
47+
48+
if err := pruneEmptyDirs(baseDir, logger); err != nil {
49+
return errors.Wrap(err, "failed to prune empty dirs")
50+
}
51+
52+
if (options & PruneUnusedPackages) != 0 {
53+
if l == nil {
54+
return errors.New("pruning unused packages requires passing a non-nil Lock")
55+
}
56+
if err := pruneUnusedPackages(baseDir, l, logger); err != nil {
57+
return errors.Wrap(err, "failed to prune unused packages")
58+
}
59+
}
60+
61+
if (options & PruneNonGoFiles) != 0 {
62+
if err := pruneNonGoFiles(baseDir, logger); err != nil {
63+
return errors.Wrap(err, "failed to prune non-Go files")
64+
}
65+
}
66+
67+
if (options & PruneGoTestFiles) != 0 {
68+
if err := pruneGoTestFiles(baseDir, logger); err != nil {
69+
return errors.Wrap(err, "failed to prune Go test files")
70+
}
71+
}
72+
73+
// Delete all empty directories.
74+
if err := pruneEmptyDirs(baseDir, logger); err != nil {
75+
return errors.Wrap(err, "failed to prune empty dirs")
76+
}
77+
78+
return nil
79+
}
80+
81+
func pruneNestedVendorDirs(baseDir string) error {
82+
return filepath.Walk(baseDir, stripNestedVendorDirs(baseDir))
83+
}
84+
85+
func pruneUnusedPackages(baseDir string, l Lock, logger *log.Logger) error {
86+
unused, err := calculateUnusedPackages(baseDir, l, logger)
87+
if err != nil {
88+
return err
89+
}
90+
91+
for _, pkg := range unused {
92+
pkgPath := filepath.Join(baseDir, pkg)
93+
94+
files, err := ioutil.ReadDir(pkgPath)
95+
if err != nil {
96+
// TODO(ibrasho) Handle this error properly.
97+
// It happens when attempting to ioutil.ReadDir a submodule.
98+
continue
99+
}
100+
101+
// Delete *.go files in the package directory.
102+
for _, file := range files {
103+
// Skip directories and files that don't have a .go suffix.
104+
if file.IsDir() || !strings.HasSuffix(file.Name(), ".go") {
105+
continue
106+
}
107+
108+
if err := os.Remove(filepath.Join(pkgPath, file.Name())); err != nil {
109+
return err
110+
}
111+
}
112+
}
113+
114+
return nil
115+
}
116+
117+
func calculateUnusedPackages(baseDir string, l Lock, logger *log.Logger) ([]string, error) {
118+
imported := calculateImportedPackages(l)
119+
sort.Strings(imported)
120+
121+
var unused []string
122+
123+
if logger != nil {
124+
logger.Println("Calculating unused packages to prune. Checking the following packages:")
125+
}
126+
127+
err := filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
128+
if err != nil {
129+
return err
130+
}
131+
132+
fmt.Println(info.Name(), info.IsDir(), info.Mode())
133+
134+
// Ignore baseDir and anything that's not a directory.
135+
if path == baseDir || !info.IsDir() {
136+
return nil
137+
}
138+
139+
pkg := strings.TrimPrefix(path, baseDir+string(filepath.Separator))
140+
if logger != nil {
141+
logger.Printf(" %s", pkg)
142+
}
143+
144+
// If pkg is not a parent of an imported package, add it to the unused list.
145+
i := sort.Search(len(imported), func(i int) bool {
146+
return pkg <= imported[i]
147+
})
148+
if i >= len(imported) || !strings.HasPrefix(imported[i], pkg) {
149+
unused = append(unused, path)
150+
}
151+
152+
return nil
153+
})
154+
fmt.Println("err", err)
155+
156+
return unused, err
157+
}
158+
159+
func calculateImportedPackages(l Lock) []string {
160+
var imported []string
161+
162+
for _, project := range l.Projects() {
163+
projectRoot := string(project.Ident().ProjectRoot)
164+
for _, pkg := range project.Packages() {
165+
imported = append(imported, filepath.Join(projectRoot, pkg))
166+
}
167+
}
168+
return imported
169+
}
170+
171+
func pruneNonGoFiles(baseDir string, logger *log.Logger) error {
172+
files, err := calculateNonGoFiles(baseDir)
173+
if err != nil {
174+
return errors.Wrap(err, "could not prune non-Go files")
175+
}
176+
177+
if err := deleteFiles(files); err != nil {
178+
return err
179+
}
180+
181+
return nil
182+
}
183+
184+
func calculateNonGoFiles(baseDir string) ([]string, error) {
185+
var files []string
186+
187+
err := filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
188+
if err != nil {
189+
return err
190+
}
191+
192+
// Ignore directories.
193+
if info.IsDir() {
194+
return nil
195+
}
196+
197+
// Ignore all Go files.
198+
if strings.HasSuffix(info.Name(), ".go") {
199+
return nil
200+
}
201+
202+
// Ignore preserved non-Go files. We check for prefix incase the file
203+
// has an extension. For example: LICENSE.md.
204+
for _, prefix := range preservedNonGoFiles {
205+
if strings.HasPrefix(info.Name(), prefix) {
206+
return nil
207+
}
208+
}
209+
210+
files = append(files, path)
211+
212+
return nil
213+
})
214+
215+
return files, err
216+
}
217+
218+
func pruneGoTestFiles(baseDir string, logger *log.Logger) error {
219+
files, err := calculateGoTestFiles(baseDir)
220+
if err != nil {
221+
return errors.Wrap(err, "could not prune Go test files")
222+
}
223+
224+
if err := deleteFiles(files); err != nil {
225+
return err
226+
}
227+
228+
return nil
229+
}
230+
231+
func calculateGoTestFiles(baseDir string) ([]string, error) {
232+
var files []string
233+
234+
err := filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
235+
if err != nil {
236+
return err
237+
}
238+
239+
// Ignore directories.
240+
if info.IsDir() {
241+
return nil
242+
}
243+
244+
// Ignore any files that is not a Go test file.
245+
if !strings.HasSuffix(info.Name(), "_test.go") {
246+
return nil
247+
}
248+
249+
files = append(files, path)
250+
251+
return nil
252+
})
253+
254+
return files, err
255+
}
256+
257+
func deleteFiles(paths []string) error {
258+
for _, path := range paths {
259+
if err := os.Remove(path); err != nil {
260+
return err
261+
}
262+
}
263+
return nil
264+
}
265+
266+
func pruneEmptyDirs(baseDir string, logger *log.Logger) error {
267+
empty, err := calculateEmptyDirs(baseDir)
268+
if err != nil {
269+
return err
270+
}
271+
272+
if logger != nil {
273+
logger.Println("Deleting empty directories:")
274+
}
275+
276+
for _, dir := range empty {
277+
if logger != nil {
278+
logger.Printf(" %s\n", strings.TrimPrefix(dir, baseDir+string(os.PathSeparator)))
279+
}
280+
}
281+
for _, dir := range empty {
282+
if err := os.Remove(dir); err != nil {
283+
return err
284+
}
285+
}
286+
287+
return nil
288+
}
289+
func calculateEmptyDirs(baseDir string) ([]string, error) {
290+
var empty []string
291+
292+
err := filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
293+
if err != nil {
294+
return nil
295+
}
296+
297+
if !info.IsDir() {
298+
return nil
299+
}
300+
301+
// TODO(ibrasho) should we use fs.IsNonEmptyDir instead?
302+
files, err := ioutil.ReadDir(path)
303+
if err != nil {
304+
return err
305+
}
306+
307+
if len(files) == 0 {
308+
empty = append(empty, path)
309+
}
310+
311+
return nil
312+
})
313+
314+
return empty, err
315+
}

‎internal/gps/strip_vendor.go

+35-17
Original file line numberDiff line numberDiff line change
@@ -6,25 +6,43 @@
66

77
package gps
88

9-
import "os"
10-
11-
func stripVendor(path string, info os.FileInfo, err error) error {
12-
if info.Name() == "vendor" {
13-
if _, err := os.Lstat(path); err == nil {
14-
if (info.Mode() & os.ModeSymlink) != 0 {
15-
realInfo, err := os.Stat(path)
16-
if err != nil {
17-
return err
18-
}
19-
if realInfo.IsDir() {
20-
return os.Remove(path)
21-
}
9+
import (
10+
"os"
11+
"path/filepath"
12+
)
13+
14+
func stripNestedVendorDirs(baseDir string) filepath.WalkFunc {
15+
return func(path string, info os.FileInfo, err error) error {
16+
// Ignore anything that's not named "vendor".
17+
if info.Name() != "vendor" {
18+
return nil
19+
}
20+
21+
// Ignore the base vendor directory.
22+
if path == baseDir {
23+
return nil
24+
}
25+
26+
// If it's a directory, delete it along with its content.
27+
if info.IsDir() {
28+
return removeAll(path)
29+
}
30+
31+
if _, err := os.Lstat(path); err != nil {
32+
return nil
33+
}
34+
35+
// If it is a symlink, check if the target is a directory and delete that instead.
36+
if (info.Mode() & os.ModeSymlink) != 0 {
37+
realInfo, err := os.Stat(path)
38+
if err != nil {
39+
return err
2240
}
23-
if info.IsDir() {
24-
return removeAll(path)
41+
if realInfo.IsDir() {
42+
return os.Remove(path)
2543
}
2644
}
27-
}
2845

29-
return nil
46+
return nil
47+
}
3048
}

‎internal/gps/strip_vendor_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ func stripVendorTestCase(tc fsTestCase) func(*testing.T) {
2727

2828
tc.before.setup(t)
2929

30-
if err := filepath.Walk(tempDir, stripVendor); err != nil {
30+
if err := filepath.Walk(tempDir, stripNestedVendorDirs(tempDir)); err != nil {
3131
t.Errorf("filepath.Walk err=%q", err)
3232
}
3333

‎internal/gps/strip_vendor_windows.go

+42-30
Original file line numberDiff line numberDiff line change
@@ -9,37 +9,49 @@ import (
99
"path/filepath"
1010
)
1111

12-
func stripVendor(path string, info os.FileInfo, err error) error {
13-
if info.Name() == "vendor" {
14-
if _, err := os.Lstat(path); err == nil {
15-
symlink := (info.Mode() & os.ModeSymlink) != 0
16-
dir := info.IsDir()
17-
18-
switch {
19-
case symlink && dir:
20-
// This could be a windows junction directory. Support for these in the
21-
// standard library is spotty, and we could easily delete an important
22-
// folder if we called os.Remove or os.RemoveAll. Just skip these.
23-
//
24-
// TODO: If we could distinguish between junctions and Windows symlinks,
25-
// we might be able to safely delete symlinks, even though junctions are
26-
// dangerous.
27-
return filepath.SkipDir
28-
29-
case symlink:
30-
realInfo, err := os.Stat(path)
31-
if err != nil {
32-
return err
33-
}
34-
if realInfo.IsDir() {
35-
return os.Remove(path)
36-
}
37-
38-
case dir:
39-
return removeAll(path)
12+
func stripNestedVendorDirs(baseDir string) filepath.WalkFunc {
13+
return func(path string, info os.FileInfo, err error) error {
14+
// Ignore anything that's not a directory or not named "vendor".
15+
if !info.IsDir() || info.Name() != "vendor" {
16+
return nil
17+
}
18+
19+
// Ignore the base vendor directory.
20+
if path == baseDir {
21+
return nil
22+
}
23+
24+
if _, err := os.Lstat(path); err != nil {
25+
return nil
26+
}
27+
28+
symlink := (info.Mode() & os.ModeSymlink) != 0
29+
dir := info.IsDir()
30+
31+
switch {
32+
case symlink && dir:
33+
// This could be a windows junction directory. Support for these in the
34+
// standard library is spotty, and we could easily delete an important
35+
// folder if we called os.Remove or os.RemoveAll. Just skip these.
36+
//
37+
// TODO: If we could distinguish between junctions and Windows symlinks,
38+
// we might be able to safely delete symlinks, even though junctions are
39+
// dangerous.
40+
return filepath.SkipDir
41+
42+
case symlink:
43+
realInfo, err := os.Stat(path)
44+
if err != nil {
45+
return err
46+
}
47+
if realInfo.IsDir() {
48+
return os.Remove(path)
4049
}
50+
51+
case dir:
52+
return removeAll(path)
4153
}
42-
}
4354

44-
return nil
55+
return nil
56+
}
4557
}

0 commit comments

Comments
 (0)
This repository has been archived.