feat: cli option to enable the new action cache (#1954)

* Enable the new action cache

* fix

* fix: CopyTarStream (Docker)

* suppress panic in test

* add a cli option for opt in

* fixups

* add package

* fix

* rc.Config nil in test???

* add feature flag

* patch

* Fix respect --action-cache-path

Co-authored-by: Björn Brauer <[email protected]>

* add remote reusable workflow to ActionCache

* fixup

---------

Co-authored-by: Björn Brauer <[email protected]>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

Author: 3 people

cmd/input.go

@@ -57,6 +57,7 @@ type Input struct {
 	actionCachePath                    string
 	logPrefixJobID                     bool
 	networkName                        string
+	useNewActionCache                  bool
 }
 
 func (i *Input) resolve(path string) string {

cmd/root.go

@@ -98,6 +98,7 @@ func Execute(ctx context.Context, version string) {
 	rootCmd.PersistentFlags().Uint16VarP(&input.cacheServerPort, "cache-server-port", "", 0, "Defines the port where the artifact server listens. 0 means a randomly available port.")
 	rootCmd.PersistentFlags().StringVarP(&input.actionCachePath, "action-cache-path", "", filepath.Join(CacheHomeDir, "act"), "Defines the path where the actions get cached and host workspaces created.")
 	rootCmd.PersistentFlags().StringVarP(&input.networkName, "network", "", "host", "Sets a docker network name. Defaults to host.")
+	rootCmd.PersistentFlags().BoolVarP(&input.useNewActionCache, "use-new-action-cache", "", false, "Enable using the new Action Cache for storing Actions locally")
 	rootCmd.SetArgs(args())
 
 	if err := rootCmd.Execute(); err != nil {
@@ -617,6 +618,11 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str
 			Matrix:                             matrixes,
 			ContainerNetworkMode:               docker_container.NetworkMode(input.networkName),
 		}
+		if input.useNewActionCache {
+			config.ActionCache = &runner.GoGitActionCache{
+				Path: config.ActionCacheDir,
+			}
+		}
 		r, err := runner.New(config)
 		if err != nil {
 			return err

pkg/container/docker_run.go

@@ -671,10 +671,28 @@ func (cr *containerReference) waitForCommand(ctx context.Context, isTerminal boo
 }
 
 func (cr *containerReference) CopyTarStream(ctx context.Context, destPath string, tarStream io.Reader) error {
-	err := cr.cli.CopyToContainer(ctx, cr.id, destPath, tarStream, types.CopyToContainerOptions{})
+	// Mkdir
+	buf := &bytes.Buffer{}
+	tw := tar.NewWriter(buf)
+	_ = tw.WriteHeader(&tar.Header{
+		Name:     destPath,
+		Mode:     777,
+		Typeflag: tar.TypeDir,
+	})
+	tw.Close()
+	err := cr.cli.CopyToContainer(ctx, cr.id, "/", buf, types.CopyToContainerOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to mkdir to copy content to container: %w", err)
+	}
+	// Copy Content
+	err = cr.cli.CopyToContainer(ctx, cr.id, destPath, tarStream, types.CopyToContainerOptions{})
 	if err != nil {
 		return fmt.Errorf("failed to copy content to container: %w", err)
 	}
+	// If this fails, then folders have wrong permissions on non root container
+	if cr.UID != 0 || cr.GID != 0 {
+		_ = cr.Exec([]string{"chown", "-R", fmt.Sprintf("%d:%d", cr.UID, cr.GID), destPath}, nil, "0", "")(ctx)
+	}
 	return nil
 }
 

pkg/model/planner.go

@@ -148,12 +148,10 @@ func NewWorkflowPlanner(path string, noWorkflowRecurse bool) (WorkflowPlanner, e
 				workflow.Name = wf.workflowDirEntry.Name()
 			}
 
-			jobNameRegex := regexp.MustCompile(`^([[:alpha:]_][[:alnum:]_\-]*)$`)
-			for k := range workflow.Jobs {
-				if ok := jobNameRegex.MatchString(k); !ok {
-					_ = f.Close()
-					return nil, fmt.Errorf("workflow is not valid. '%s': Job name '%s' is invalid. Names must start with a letter or '_' and contain only alphanumeric characters, '-', or '_'", workflow.Name, k)
-				}
+			err = validateJobName(workflow)
+			if err != nil {
+				_ = f.Close()
+				return nil, err
 			}
 
 			wp.workflows = append(wp.workflows, workflow)
@@ -164,6 +162,42 @@ func NewWorkflowPlanner(path string, noWorkflowRecurse bool) (WorkflowPlanner, e
 	return wp, nil
 }
 
+func NewSingleWorkflowPlanner(name string, f io.Reader) (WorkflowPlanner, error) {
+	wp := new(workflowPlanner)
+
+	log.Debugf("Reading workflow %s", name)
+	workflow, err := ReadWorkflow(f)
+	if err != nil {
+		if err == io.EOF {
+			return nil, fmt.Errorf("unable to read workflow '%s': file is empty: %w", name, err)
+		}
+		return nil, fmt.Errorf("workflow is not valid. '%s': %w", name, err)
+	}
+	workflow.File = name
+	if workflow.Name == "" {
+		workflow.Name = name
+	}
+
+	err = validateJobName(workflow)
+	if err != nil {
+		return nil, err
+	}
+
+	wp.workflows = append(wp.workflows, workflow)
+
+	return wp, nil
+}
+
+func validateJobName(workflow *Workflow) error {
+	jobNameRegex := regexp.MustCompile(`^([[:alpha:]_][[:alnum:]_\-]*)$`)
+	for k := range workflow.Jobs {
+		if ok := jobNameRegex.MatchString(k); !ok {
+			return fmt.Errorf("workflow is not valid. '%s': Job name '%s' is invalid. Names must start with a letter or '_' and contain only alphanumeric characters, '-', or '_'", workflow.Name, k)
+		}
+	}
+	return nil
+}
+
 type workflowPlanner struct {
 	workflows []*Workflow
 }

pkg/runner/action.go

@@ -44,7 +44,7 @@ func readActionImpl(ctx context.Context, step *model.Step, actionDir string, act
 	reader, closer, err := readFile("action.yml")
 	if os.IsNotExist(err) {
 		reader, closer, err = readFile("action.yaml")
-		if err != nil {
+		if os.IsNotExist(err) {
 			if _, closer, err2 := readFile("Dockerfile"); err2 == nil {
 				closer.Close()
 				action := &model.Action{
@@ -91,6 +91,8 @@ func readActionImpl(ctx context.Context, step *model.Step, actionDir string, act
 				}
 			}
 			return nil, err
+		} else if err != nil {
+			return nil, err
 		}
 	} else if err != nil {
 		return nil, err
@@ -110,6 +112,17 @@ func maybeCopyToActionDir(ctx context.Context, step actionStep, actionDir string
 	if stepModel.Type() != model.StepTypeUsesActionRemote {
 		return nil
 	}
+
+	if rc.Config != nil && rc.Config.ActionCache != nil {
+		raction := step.(*stepActionRemote)
+		ta, err := rc.Config.ActionCache.GetTarArchive(ctx, raction.cacheDir, raction.resolvedSha, "")
+		if err != nil {
+			return err
+		}
+		defer ta.Close()
+		return rc.JobContainer.CopyTarStream(ctx, containerActionDir, ta)
+	}
+
 	if err := removeGitIgnore(ctx, actionDir); err != nil {
 		return err
 	}
@@ -265,6 +278,13 @@ func execAsDocker(ctx context.Context, step actionStep, actionName string, based
 					return err
 				}
 				defer buildContext.Close()
+			} else if rc.Config.ActionCache != nil {
+				rstep := step.(*stepActionRemote)
+				buildContext, err = rc.Config.ActionCache.GetTarArchive(ctx, rstep.cacheDir, rstep.resolvedSha, contextDir)
+				if err != nil {
+					return err
+				}
+				defer buildContext.Close()
 			}
 			prepImage = container.NewDockerBuildExecutor(container.NewDockerBuildExecutorInput{
 				ContextDir:   contextDir,

pkg/runner/reusable_workflow.go

@@ -1,6 +1,7 @@
 package runner
 
 import (
+	"archive/tar"
 	"context"
 	"errors"
 	"fmt"
@@ -33,12 +34,51 @@ func newRemoteReusableWorkflowExecutor(rc *RunContext) common.Executor {
 	filename := fmt.Sprintf("%s/%s@%s", remoteReusableWorkflow.Org, remoteReusableWorkflow.Repo, remoteReusableWorkflow.Ref)
 	workflowDir := fmt.Sprintf("%s/%s", rc.ActionCacheDir(), safeFilename(filename))
 
+	if rc.Config.ActionCache != nil {
+		return newActionCacheReusableWorkflowExecutor(rc, filename, remoteReusableWorkflow)
+	}
+
 	return common.NewPipelineExecutor(
 		newMutexExecutor(cloneIfRequired(rc, *remoteReusableWorkflow, workflowDir)),
 		newReusableWorkflowExecutor(rc, workflowDir, fmt.Sprintf("./.github/workflows/%s", remoteReusableWorkflow.Filename)),
 	)
 }
 
+func newActionCacheReusableWorkflowExecutor(rc *RunContext, filename string, remoteReusableWorkflow *remoteReusableWorkflow) common.Executor {
+	return func(ctx context.Context) error {
+		ghctx := rc.getGithubContext(ctx)
+		remoteReusableWorkflow.URL = ghctx.ServerURL
+		sha, err := rc.Config.ActionCache.Fetch(ctx, filename, remoteReusableWorkflow.CloneURL(), remoteReusableWorkflow.Ref, ghctx.Token)
+		if err != nil {
+			return err
+		}
+		archive, err := rc.Config.ActionCache.GetTarArchive(ctx, filename, sha, fmt.Sprintf(".github/workflows/%s", remoteReusableWorkflow.Filename))
+		if err != nil {
+			return err
+		}
+		defer archive.Close()
+		treader := tar.NewReader(archive)
+		if _, err = treader.Next(); err != nil {
+			return err
+		}
+		planner, err := model.NewSingleWorkflowPlanner(remoteReusableWorkflow.Filename, treader)
+		if err != nil {
+			return err
+		}
+		plan, err := planner.PlanEvent("workflow_call")
+		if err != nil {
+			return err
+		}
+
+		runner, err := NewReusableWorkflowRunner(rc)
+		if err != nil {
+			return err
+		}
+
+		return runner.NewPlanExecutor(plan)(ctx)
+	}
+}
+
 var (
 	executorLock sync.Mutex
 )

pkg/runner/runner.go

@@ -59,6 +59,7 @@ type Config struct {
 	ReplaceGheActionTokenWithGithubCom string                       // Token of private action repo on GitHub.
 	Matrix                             map[string]map[string]bool   // Matrix config to run
 	ContainerNetworkMode               docker_container.NetworkMode // the network mode of job containers (the value of --network)
+	ActionCache                        ActionCache                  // Use a custom ActionCache Implementation
 }
 
 type caller struct {

pkg/runner/step.go

@@ -34,6 +34,9 @@ const (
 	stepStagePost
 )
 
+// Controls how many symlinks are resolved for local and remote Actions
+const maxSymlinkDepth = 10
+
 func (s stepStage) String() string {
 	switch s {
 	case stepStagePre:
@@ -307,3 +310,13 @@ func mergeIntoMapCaseInsensitive(target map[string]string, maps ...map[string]st
 		}
 	}
 }
+
+func symlinkJoin(filename, sym, parent string) (string, error) {
+	dir := path.Dir(filename)
+	dest := path.Join(dir, sym)
+	prefix := path.Clean(parent) + "/"
+	if strings.HasPrefix(dest, prefix) || prefix == "./" {
+		return dest, nil
+	}
+	return "", fmt.Errorf("symlink tries to access file '%s' outside of '%s'", strings.ReplaceAll(dest, "'", "''"), strings.ReplaceAll(parent, "'", "''"))
+}

pkg/runner/step_action_local.go

@@ -3,7 +3,10 @@ package runner
 import (
 	"archive/tar"
 	"context"
+	"errors"
+	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path"
 	"path/filepath"
@@ -42,15 +45,31 @@ func (sal *stepActionLocal) main() common.Executor {
 		localReader := func(ctx context.Context) actionYamlReader {
 			_, cpath := getContainerActionPaths(sal.Step, path.Join(actionDir, ""), sal.RunContext)
 			return func(filename string) (io.Reader, io.Closer, error) {
-				tars, err := sal.RunContext.JobContainer.GetContainerArchive(ctx, path.Join(cpath, filename))
-				if err != nil {
-					return nil, nil, os.ErrNotExist
+				spath := path.Join(cpath, filename)
+				for i := 0; i < maxSymlinkDepth; i++ {
+					tars, err := sal.RunContext.JobContainer.GetContainerArchive(ctx, spath)
+					if errors.Is(err, fs.ErrNotExist) {
+						return nil, nil, err
+					} else if err != nil {
+						return nil, nil, fs.ErrNotExist
+					}
+					treader := tar.NewReader(tars)
+					header, err := treader.Next()
+					if errors.Is(err, io.EOF) {
+						return nil, nil, os.ErrNotExist
+					} else if err != nil {
+						return nil, nil, err
+					}
+					if header.FileInfo().Mode()&os.ModeSymlink == os.ModeSymlink {
+						spath, err = symlinkJoin(spath, header.Linkname, cpath)
+						if err != nil {
+							return nil, nil, err
+						}
+					} else {
+						return treader, tars, nil
+					}
 				}
-				treader := tar.NewReader(tars)
-				if _, err := treader.Next(); err != nil {
-					return nil, nil, os.ErrNotExist
-				}
-				return treader, tars, nil
+				return nil, nil, fmt.Errorf("max depth %d of symlinks exceeded while reading %s", maxSymlinkDepth, spath)
 			}
 		}
 

pkg/runner/step_action_remote.go

@@ -1,6 +1,7 @@
 package runner
 
 import (
+	"archive/tar"
 	"context"
 	"errors"
 	"fmt"
@@ -28,6 +29,8 @@ type stepActionRemote struct {
 	action              *model.Action
 	env                 map[string]string
 	remoteAction        *remoteAction
+	cacheDir            string
+	resolvedSha         string
 }
 
 var (
@@ -60,6 +63,46 @@ func (sar *stepActionRemote) prepareActionExecutor() common.Executor {
 				github.Token = sar.RunContext.Config.ReplaceGheActionTokenWithGithubCom
 			}
 		}
+		if sar.RunContext.Config.ActionCache != nil {
+			cache := sar.RunContext.Config.ActionCache
+
+			var err error
+			sar.cacheDir = fmt.Sprintf("%s/%s", sar.remoteAction.Org, sar.remoteAction.Repo)
+			sar.resolvedSha, err = cache.Fetch(ctx, sar.cacheDir, sar.remoteAction.URL+"/"+sar.cacheDir, sar.remoteAction.Ref, github.Token)
+			if err != nil {
+				return err
+			}
+
+			remoteReader := func(ctx context.Context) actionYamlReader {
+				return func(filename string) (io.Reader, io.Closer, error) {
+					spath := filename
+					for i := 0; i < maxSymlinkDepth; i++ {
+						tars, err := cache.GetTarArchive(ctx, sar.cacheDir, sar.resolvedSha, spath)
+						if err != nil {
+							return nil, nil, os.ErrNotExist
+						}
+						treader := tar.NewReader(tars)
+						header, err := treader.Next()
+						if err != nil {
+							return nil, nil, os.ErrNotExist
+						}
+						if header.FileInfo().Mode()&os.ModeSymlink == os.ModeSymlink {
+							spath, err = symlinkJoin(spath, header.Linkname, ".")
+							if err != nil {
+								return nil, nil, err
+							}
+						} else {
+							return treader, tars, nil
+						}
+					}
+					return nil, nil, fmt.Errorf("max depth %d of symlinks exceeded while reading %s", maxSymlinkDepth, spath)
+				}
+			}
+
+			actionModel, err := sar.readAction(ctx, sar.Step, sar.resolvedSha, sar.remoteAction.Path, remoteReader(ctx), os.WriteFile)
+			sar.action = actionModel
+			return err
+		}
 
 		actionDir := fmt.Sprintf("%s/%s", sar.RunContext.ActionCacheDir(), safeFilename(sar.Step.Uses))
 		gitClone := stepActionRemoteNewCloneExecutor(git.NewGitCloneExecutorInput{