From 418b0db047fd23799bf1bd04c8dd68fd079c5d2a Mon Sep 17 00:00:00 2001 From: Casey Lee Date: Thu, 20 Feb 2020 22:43:20 -0500 Subject: [PATCH 01/14] reuse containers to fix #86 --- README.md | 6 ++-- cmd/platforms.go | 6 ++-- pkg/model/workflow.go | 1 + pkg/runner/run_context.go | 35 ++++++++++++++++++---- pkg/runner/step.go | 49 ++++++++++++++++++++++++------- pkg/runner/testdata/node/push.yml | 16 ++++++++++ 6 files changed, 91 insertions(+), 22 deletions(-) create mode 100644 pkg/runner/testdata/node/push.yml diff --git a/README.md b/README.md index bd1a4609..813a3b81 100644 --- a/README.md +++ b/README.md @@ -56,9 +56,9 @@ GitHub Actions offers managed [virtual environments](https://help.github.com/en/ | GitHub Runner | Docker Image | | --------------- | ------------ | -| ubuntu-latest | [ubuntu:18.04](https://hub.docker.com/_/ubuntu) | -| ubuntu-18.04 | [ubuntu:18.04](https://hub.docker.com/_/ubuntu) | -| ubuntu-16.04 | [ubuntu:16.04](https://hub.docker.com/_/ubuntu) | +| ubuntu-latest | [node:12.6-buster-slim](https://hub.docker.com/_/buildpack-deps) | +| ubuntu-18.04 | [node:12.6-buster-slim](https://hub.docker.com/_/buildpack-deps) | +| ubuntu-16.04 | [node:12.6-stretch-slim](https://hub.docker.com/_/buildpack-deps) | | windows-latest | `unsupported` | | windows-2019 | `unsupported` | | macos-latest | `unsupported` | diff --git a/cmd/platforms.go b/cmd/platforms.go index 46d45b19..b79e8edd 100644 --- a/cmd/platforms.go +++ b/cmd/platforms.go @@ -6,9 +6,9 @@ import ( func (i *Input) newPlatforms() map[string]string { platforms := map[string]string{ - "ubuntu-latest": "ubuntu:18.04", - "ubuntu-18.04": "ubuntu:18.04", - "ubuntu-16.04": "ubuntu:16.04", + "ubuntu-latest": "node:12.6-buster-slim", + "ubuntu-18.04": "node:12.6-buster-slim", + "ubuntu-16.04": "node:12.6-stretch-slim", "windows-latest": "", "windows-2019": "", "macos-latest": "", diff --git a/pkg/model/workflow.go b/pkg/model/workflow.go index c25cf5cd..f0c2a462 100644 --- a/pkg/model/workflow.go +++ b/pkg/model/workflow.go @@ -104,6 +104,7 @@ type ContainerSpec struct { Entrypoint string Args string Name string + Reuse bool } // Step is the structure of one step in a job diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go index 2289847a..3465367f 100644 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -9,7 +9,6 @@ import ( "io" "io/ioutil" "os" - "path/filepath" "regexp" "runtime" "strings" @@ -107,7 +106,19 @@ func (rc *RunContext) Executor() common.Executor { return nil } - return common.NewPipelineExecutor(steps...)(ctx) + nullLogger := logrus.New() + nullLogger.Out = ioutil.Discard + if !rc.Config.ReuseContainers { + rc.newContainerCleaner()(common.WithLogger(ctx, nullLogger)) + } + + err := common.NewPipelineExecutor(steps...)(ctx) + + if !rc.Config.ReuseContainers { + rc.newContainerCleaner()(common.WithLogger(ctx, nullLogger)) + } + + return err } } @@ -202,7 +213,7 @@ func (rc *RunContext) runContainer(containerSpec *model.ContainerSpec) common.Ex fmt.Sprintf("%s:%s", "/var/run/docker.sock", "/var/run/docker.sock"), }, Content: map[string]io.Reader{"/github": ghReader}, - ReuseContainers: rc.Config.ReuseContainers, + ReuseContainers: containerSpec.Reuse, Stdout: logWriter, Stderr: logWriter, })(ctx) @@ -241,17 +252,29 @@ func (rc *RunContext) createGithubTarball() (io.Reader, error) { } -func (rc *RunContext) createContainerName(stepID string) string { - containerName := fmt.Sprintf("%s-%s", stepID, rc.Tempdir) +func (rc *RunContext) createContainerName() string { + containerName := rc.Run.String() containerName = regexp.MustCompile("[^a-zA-Z0-9]").ReplaceAllString(containerName, "-") - prefix := fmt.Sprintf("%s-", trimToLen(filepath.Base(rc.Config.Workdir), 10)) + prefix := "" suffix := "" containerName = trimToLen(containerName, 30-(len(prefix)+len(suffix))) return fmt.Sprintf("%s%s%s", prefix, containerName, suffix) + +} + +func (rc *RunContext) createStepContainerName(stepID string) string { + + prefix := regexp.MustCompile("[^a-zA-Z0-9]").ReplaceAllString(rc.createContainerName(), "-") + suffix := regexp.MustCompile("[^a-zA-Z0-9]").ReplaceAllString(stepID, "-") + containerName := trimToLen(prefix, 30-len(suffix)) + return fmt.Sprintf("%s%s%s", prefix, containerName, suffix) } func trimToLen(s string, l int) string { + if l < 0 { + l = 0 + } if len(s) > l { return s[:l] } diff --git a/pkg/runner/step.go b/pkg/runner/step.go index 408f0330..e5cf61c4 100644 --- a/pkg/runner/step.go +++ b/pkg/runner/step.go @@ -37,21 +37,44 @@ func (rc *RunContext) setupEnv(containerSpec *model.ContainerSpec, step *model.S } } +func (rc *RunContext) newContainerCleaner() common.Executor { + job := rc.Run.Job() + containerSpec := new(model.ContainerSpec) + containerSpec.Name = rc.createContainerName() + containerSpec.Reuse = false + + if job.Container != nil { + containerSpec.Image = job.Container.Image + } else { + platformName := rc.ExprEval.Interpolate(rc.Run.Job().RunsOn) + containerSpec.Image = rc.Config.Platforms[strings.ToLower(platformName)] + } + containerSpec.Entrypoint = "bash --noprofile --norc -o pipefail -c echo 'cleaning up'" + return common.NewPipelineExecutor( + rc.pullImage(containerSpec), + rc.runContainer(containerSpec), + ) +} + func (rc *RunContext) newStepExecutor(step *model.Step) common.Executor { job := rc.Run.Job() containerSpec := new(model.ContainerSpec) - containerSpec.Name = rc.createContainerName(step.ID) + containerSpec.Name = rc.createContainerName() + containerSpec.Reuse = true + + if job.Container != nil { + containerSpec.Image = job.Container.Image + } else { + platformName := rc.ExprEval.Interpolate(rc.Run.Job().RunsOn) + containerSpec.Image = rc.Config.Platforms[strings.ToLower(platformName)] + } switch step.Type() { case model.StepTypeRun: if job.Container != nil { - containerSpec.Image = job.Container.Image containerSpec.Ports = job.Container.Ports containerSpec.Volumes = job.Container.Volumes containerSpec.Options = job.Container.Options - } else { - platformName := rc.ExprEval.Interpolate(rc.Run.Job().RunsOn) - containerSpec.Image = rc.Config.Platforms[strings.ToLower(platformName)] } return common.NewPipelineExecutor( rc.setupEnv(containerSpec, step), @@ -62,8 +85,10 @@ func (rc *RunContext) newStepExecutor(step *model.Step) common.Executor { case model.StepTypeUsesDockerURL: containerSpec.Image = strings.TrimPrefix(step.Uses, "docker://") + containerSpec.Name = rc.createStepContainerName(step.ID) containerSpec.Entrypoint = step.With["entrypoint"] containerSpec.Args = step.With["args"] + containerSpec.Reuse = rc.Config.ReuseContainers return common.NewPipelineExecutor( rc.setupEnv(containerSpec, step), rc.pullImage(containerSpec), @@ -71,7 +96,6 @@ func (rc *RunContext) newStepExecutor(step *model.Step) common.Executor { ) case model.StepTypeUsesActionLocal: - containerSpec.Image = fmt.Sprintf("%s:%s", containerSpec.Name, "latest") return common.NewPipelineExecutor( rc.setupEnv(containerSpec, step), rc.setupAction(containerSpec, filepath.Join(rc.Config.Workdir, step.Uses)), @@ -91,7 +115,6 @@ func (rc *RunContext) newStepExecutor(step *model.Step) common.Executor { if err != nil { return common.NewErrorExecutor(err) } - containerSpec.Image = fmt.Sprintf("%s:%s", remoteAction.Repo, remoteAction.Ref) return common.NewPipelineExecutor( common.NewGitCloneExecutor(common.NewGitCloneExecutorInput{ URL: remoteAction.CloneURL(), @@ -194,18 +217,24 @@ func (rc *RunContext) setupAction(containerSpec *model.ContainerSpec, actionDir switch action.Runs.Using { case model.ActionRunsUsingNode12: - containerSpec.Image = "node:12-alpine" if strings.HasPrefix(actionDir, rc.Config.Workdir) { - containerSpec.Args = fmt.Sprintf("node /github/workspace/%s/%s", strings.TrimPrefix(actionDir, rc.Config.Workdir), action.Runs.Main) + containerSpec.Entrypoint = fmt.Sprintf("node /github/workspace/%s/%s", strings.TrimPrefix(actionDir, rc.Config.Workdir), action.Runs.Main) } else if strings.HasPrefix(actionDir, rc.Tempdir) { - containerSpec.Args = fmt.Sprintf("node /github/home/%s/%s", strings.TrimPrefix(actionDir, rc.Tempdir), action.Runs.Main) + containerSpec.Entrypoint = fmt.Sprintf("node /github/home/%s/%s", strings.TrimPrefix(actionDir, rc.Tempdir), action.Runs.Main) } case model.ActionRunsUsingDocker: + if strings.HasPrefix(actionDir, rc.Config.Workdir) { + containerSpec.Name = rc.createStepContainerName(strings.TrimPrefix(actionDir, rc.Config.Workdir)) + } else if strings.HasPrefix(actionDir, rc.Tempdir) { + containerSpec.Name = rc.createStepContainerName(strings.TrimPrefix(actionDir, rc.Tempdir)) + } + containerSpec.Reuse = rc.Config.ReuseContainers if strings.HasPrefix(action.Runs.Image, "docker://") { containerSpec.Image = strings.TrimPrefix(action.Runs.Image, "docker://") containerSpec.Entrypoint = strings.Join(action.Runs.Entrypoint, " ") containerSpec.Args = strings.Join(action.Runs.Args, " ") } else { + containerSpec.Image = fmt.Sprintf("%s:%s", containerSpec.Name, "latest") contextDir := filepath.Join(actionDir, action.Runs.Main) return container.NewDockerBuildExecutor(container.NewDockerBuildExecutorInput{ ContextDir: contextDir, diff --git a/pkg/runner/testdata/node/push.yml b/pkg/runner/testdata/node/push.yml new file mode 100644 index 00000000..1be89469 --- /dev/null +++ b/pkg/runner/testdata/node/push.yml @@ -0,0 +1,16 @@ +name: NodeJS Test + +on: push + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v1 + with: + node-version: 12 + registry-url: https://registry.npmjs.org/ + + - name: Install Dependencies + run: npm install From 2cb276ca0517bd97675d893338c4bb3448b2a84d Mon Sep 17 00:00:00 2001 From: Casey Lee Date: Fri, 21 Feb 2020 08:42:00 -0800 Subject: [PATCH 02/14] unchecked error --- pkg/runner/run_context.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go index 3465367f..b2ff1e0e 100644 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -109,13 +109,13 @@ func (rc *RunContext) Executor() common.Executor { nullLogger := logrus.New() nullLogger.Out = ioutil.Discard if !rc.Config.ReuseContainers { - rc.newContainerCleaner()(common.WithLogger(ctx, nullLogger)) + _ = rc.newContainerCleaner()(common.WithLogger(ctx, nullLogger)) } err := common.NewPipelineExecutor(steps...)(ctx) if !rc.Config.ReuseContainers { - rc.newContainerCleaner()(common.WithLogger(ctx, nullLogger)) + _ = rc.newContainerCleaner()(common.WithLogger(ctx, nullLogger)) } return err From 9179d8924dce148c0844881646ab992533a49084 Mon Sep 17 00:00:00 2001 From: Casey Lee Date: Fri, 21 Feb 2020 22:19:59 -0800 Subject: [PATCH 03/14] updates for tests --- pkg/runner/run_context.go | 5 +++-- pkg/runner/runner_test.go | 4 ++-- pkg/runner/testdata/node/push.yml | 3 ++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go index b2ff1e0e..5a662d65 100644 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -267,8 +267,9 @@ func (rc *RunContext) createStepContainerName(stepID string) string { prefix := regexp.MustCompile("[^a-zA-Z0-9]").ReplaceAllString(rc.createContainerName(), "-") suffix := regexp.MustCompile("[^a-zA-Z0-9]").ReplaceAllString(stepID, "-") - containerName := trimToLen(prefix, 30-len(suffix)) - return fmt.Sprintf("%s%s%s", prefix, containerName, suffix) + prefix = trimToLen(prefix, 30-(1+len(suffix))) + name := strings.Trim(fmt.Sprintf("%s-%s", prefix, suffix), "-") + return name } func trimToLen(s string, l int) string { diff --git a/pkg/runner/runner_test.go b/pkg/runner/runner_test.go index b6f2c863..e4fb880f 100644 --- a/pkg/runner/runner_test.go +++ b/pkg/runner/runner_test.go @@ -57,13 +57,13 @@ func TestRunEvent(t *testing.T) { table := table t.Run(table.workflowPath, func(t *testing.T) { platforms := map[string]string{ - "ubuntu-latest": "ubuntu:18.04", + "ubuntu-latest": "node:12.6-buster-slim", } runnerConfig := &Config{ Workdir: "testdata", EventName: table.eventName, Platforms: platforms, - ReuseContainers: true, + ReuseContainers: false, } runner, err := New(runnerConfig) assert.NilError(t, err, table.workflowPath) diff --git a/pkg/runner/testdata/node/push.yml b/pkg/runner/testdata/node/push.yml index 1be89469..0eebcf42 100644 --- a/pkg/runner/testdata/node/push.yml +++ b/pkg/runner/testdata/node/push.yml @@ -6,11 +6,12 @@ jobs: test: runs-on: ubuntu-latest steps: + - run: which node - uses: actions/checkout@v2 - uses: actions/setup-node@v1 with: node-version: 12 registry-url: https://registry.npmjs.org/ - + - run: which node - name: Install Dependencies run: npm install From 01876438c2f2c5493375019b7b033b8a67876a43 Mon Sep 17 00:00:00 2001 From: Casey Lee Date: Sun, 23 Feb 2020 15:01:25 -0800 Subject: [PATCH 04/14] shared container for job --- go.sum | 1 + pkg/common/executor.go | 9 + pkg/container/docker_pull.go | 2 +- pkg/container/docker_run.go | 228 ++++++++++++++++++---- pkg/model/workflow.go | 78 +++++++- pkg/runner/expression.go | 14 +- pkg/runner/run_context.go | 292 ++++++++++++++--------------- pkg/runner/runner.go | 64 +------ pkg/runner/step.go | 276 --------------------------- pkg/runner/testdata/basic/push.yml | 11 +- pkg/runner/testdata/node/push.yml | 1 - 11 files changed, 445 insertions(+), 531 deletions(-) delete mode 100644 pkg/runner/step.go diff --git a/go.sum b/go.sum index c747a1dc..1a0a56fd 100644 --- a/go.sum +++ b/go.sum @@ -23,6 +23,7 @@ github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BU github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/engine v0.0.0-20181106193140-f5749085e9cb h1:PyjxRdW1mqCmSoxy/6uP01P7CGbsD+woX+oOWbaUPwQ= github.com/docker/engine v0.0.0-20181106193140-f5749085e9cb/go.mod h1:3CPr2caMgTHxxIAZgEMd3uLYPDlRvPqCpyeRf6ncPcY= +github.com/docker/engine v1.13.1 h1:Cks33UT9YBW5Xyc3MtGDq2IPgqfJtJ+qkFaxc2b0Euc= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk= diff --git a/pkg/common/executor.go b/pkg/common/executor.go index f16de41d..197cd5b3 100644 --- a/pkg/common/executor.go +++ b/pkg/common/executor.go @@ -40,6 +40,15 @@ func NewInfoExecutor(format string, args ...interface{}) Executor { } } +// NewDebugExecutor is an executor that logs messages +func NewDebugExecutor(format string, args ...interface{}) Executor { + return func(ctx context.Context) error { + logger := Logger(ctx) + logger.Debugf(format, args...) + return nil + } +} + // NewPipelineExecutor creates a new executor from a series of other executors func NewPipelineExecutor(executors ...Executor) Executor { if len(executors) == 0 { diff --git a/pkg/container/docker_pull.go b/pkg/container/docker_pull.go index 7aa9b893..69a2e2f4 100644 --- a/pkg/container/docker_pull.go +++ b/pkg/container/docker_pull.go @@ -21,7 +21,7 @@ type NewDockerPullExecutorInput struct { func NewDockerPullExecutor(input NewDockerPullExecutorInput) common.Executor { return func(ctx context.Context) error { logger := common.Logger(ctx) - logger.Infof("%sdocker pull %v", logPrefix, input.Image) + logger.Debugf("%sdocker pull %v", logPrefix, input.Image) if common.Dryrun(ctx) { return nil diff --git a/pkg/container/docker_run.go b/pkg/container/docker_run.go index b54ab875..63d83923 100644 --- a/pkg/container/docker_run.go +++ b/pkg/container/docker_run.go @@ -1,6 +1,8 @@ package container import ( + "archive/tar" + "bytes" "context" "fmt" "io" @@ -8,60 +10,119 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/mount" "github.com/docker/docker/client" "github.com/docker/docker/pkg/stdcopy" "github.com/nektos/act/pkg/common" "github.com/pkg/errors" + log "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh/terminal" ) -// NewDockerRunExecutorInput the input for the NewDockerRunExecutor function -type NewDockerRunExecutorInput struct { - Image string - Entrypoint []string - Cmd []string - WorkingDir string - Env []string - Binds []string - Content map[string]io.Reader - Volumes []string - Name string - ReuseContainers bool - Stdout io.Writer - Stderr io.Writer +// NewContainerInput the input for the New function +type NewContainerInput struct { + Image string + Entrypoint []string + Cmd []string + WorkingDir string + Env []string + Binds []string + Mounts map[string]string + Name string + Stdout io.Writer + Stderr io.Writer } -// NewDockerRunExecutor function to create a run executor for the container -func NewDockerRunExecutor(input NewDockerRunExecutorInput) common.Executor { +// FileEntry is a file to copy to a container +type FileEntry struct { + Name string + Mode int64 + Body string +} + +// Container for managing docker run containers +type Container interface { + Create() common.Executor + Copy(destPath string, files ...*FileEntry) common.Executor + Pull(forcePull bool) common.Executor + Start(attach bool) common.Executor + Exec(command []string, env map[string]string) common.Executor + Remove() common.Executor +} + +// NewContainer creates a reference to a container +func NewContainer(input *NewContainerInput) Container { cr := new(containerReference) cr.input = input + return cr +} +func (cr *containerReference) Create() common.Executor { return common. - NewInfoExecutor("%sdocker run image=%s entrypoint=%+q cmd=%+q", logPrefix, input.Image, input.Entrypoint, input.Cmd). + NewDebugExecutor("%sdocker create image=%s entrypoint=%+q cmd=%+q", logPrefix, cr.input.Image, cr.input.Entrypoint, cr.input.Cmd). Then( common.NewPipelineExecutor( cr.connect(), cr.find(), - cr.remove().IfBool(!input.ReuseContainers), cr.create(), - cr.copyContent(), - cr.attach(), - cr.start(), - cr.wait(), - ).Finally( - cr.remove().IfBool(!input.ReuseContainers), ).IfNot(common.Dryrun), ) } +func (cr *containerReference) Start(attach bool) common.Executor { + return common. + NewDebugExecutor("%sdocker run image=%s entrypoint=%+q cmd=%+q", logPrefix, cr.input.Image, cr.input.Entrypoint, cr.input.Cmd). + Then( + common.NewPipelineExecutor( + cr.connect(), + cr.find(), + cr.attach().IfBool(attach), + cr.start(), + cr.wait().IfBool(attach), + ).IfNot(common.Dryrun), + ) +} +func (cr *containerReference) Pull(forcePull bool) common.Executor { + return NewDockerPullExecutor(NewDockerPullExecutorInput{ + Image: cr.input.Image, + ForcePull: forcePull, + }) +} +func (cr *containerReference) Copy(destPath string, files ...*FileEntry) common.Executor { + return common.NewPipelineExecutor( + cr.connect(), + cr.find(), + cr.copyContent(destPath, files...), + ).IfNot(common.Dryrun) +} + +func (cr *containerReference) Exec(command []string, env map[string]string) common.Executor { + + return common.NewPipelineExecutor( + cr.connect(), + cr.find(), + cr.exec(command, env), + ).IfNot(common.Dryrun) +} +func (cr *containerReference) Remove() common.Executor { + return common.NewPipelineExecutor( + cr.connect(), + cr.find(), + ).Finally( + cr.remove(), + ).IfNot(common.Dryrun) +} type containerReference struct { - input NewDockerRunExecutorInput cli *client.Client id string + input *NewContainerInput } func (cr *containerReference) connect() common.Executor { return func(ctx context.Context) error { + if cr.cli != nil { + return nil + } cli, err := client.NewClientWithOpts(client.FromEnv) if err != nil { return errors.WithStack(err) @@ -74,6 +135,9 @@ func (cr *containerReference) connect() common.Executor { func (cr *containerReference) find() common.Executor { return func(ctx context.Context) error { + if cr.id != "" { + return nil + } containers, err := cr.cli.ContainerList(ctx, types.ContainerListOptions{ All: true, }) @@ -134,15 +198,18 @@ func (cr *containerReference) create() common.Executor { Tty: isTerminal, } - if len(input.Volumes) > 0 { - config.Volumes = make(map[string]struct{}) - for _, vol := range input.Volumes { - config.Volumes[vol] = struct{}{} - } + mounts := make([]mount.Mount, 0) + for mountSource, mountTarget := range input.Mounts { + mounts = append(mounts, mount.Mount{ + Type: mount.TypeVolume, + Source: mountSource, + Target: mountTarget, + }) } resp, err := cr.cli.ContainerCreate(ctx, config, &container.HostConfig{ - Binds: input.Binds, + Binds: input.Binds, + Mounts: mounts, }, nil, input.Name) if err != nil { return errors.WithStack(err) @@ -155,15 +222,100 @@ func (cr *containerReference) create() common.Executor { } } -func (cr *containerReference) copyContent() common.Executor { +func (cr *containerReference) exec(cmd []string, env map[string]string) common.Executor { return func(ctx context.Context) error { logger := common.Logger(ctx) - for dstPath, srcReader := range cr.input.Content { - logger.Debugf("Extracting content to '%s'", dstPath) - err := cr.cli.CopyToContainer(ctx, cr.id, dstPath, srcReader, types.CopyToContainerOptions{}) - if err != nil { - return errors.WithStack(err) + logger.Debugf("Exec command '%s'", cmd) + isTerminal := terminal.IsTerminal(int(os.Stdout.Fd())) + envList := make([]string, 0) + for k, v := range env { + envList = append(envList, fmt.Sprintf("%s=%s", k, v)) + } + + idResp, err := cr.cli.ContainerExecCreate(ctx, cr.id, types.ExecConfig{ + Cmd: cmd, + WorkingDir: cr.input.WorkingDir, + Env: envList, + Tty: isTerminal, + AttachStderr: true, + AttachStdout: true, + }) + if err != nil { + return errors.WithStack(err) + } + + resp, err := cr.cli.ContainerExecAttach(ctx, idResp.ID, types.ExecStartCheck{ + Tty: isTerminal, + }) + if err != nil { + return errors.WithStack(err) + } + var outWriter io.Writer + outWriter = cr.input.Stdout + if outWriter == nil { + outWriter = os.Stdout + } + errWriter := cr.input.Stderr + if errWriter == nil { + errWriter = os.Stderr + } + + err = cr.cli.ContainerExecStart(ctx, idResp.ID, types.ExecStartCheck{ + Tty: isTerminal, + }) + if err != nil { + return errors.WithStack(err) + } + + if !isTerminal || os.Getenv("NORAW") != "" { + _, err = stdcopy.StdCopy(outWriter, errWriter, resp.Reader) + } else { + _, err = io.Copy(outWriter, resp.Reader) + } + if err != nil { + logger.Error(err) + } + + inspectResp, err := cr.cli.ContainerExecInspect(ctx, idResp.ID) + if err != nil { + return errors.WithStack(err) + } + + if inspectResp.ExitCode == 0 { + return nil + } + + return fmt.Errorf("exit with `FAILURE`: %v", inspectResp.ExitCode) + } +} + +func (cr *containerReference) copyContent(dstPath string, files ...*FileEntry) common.Executor { + return func(ctx context.Context) error { + logger := common.Logger(ctx) + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + for _, file := range files { + log.Debugf("Writing entry to tarball %s len:%d", file.Name, len(file.Body)) + hdr := &tar.Header{ + Name: file.Name, + Mode: file.Mode, + Size: int64(len(file.Body)), } + if err := tw.WriteHeader(hdr); err != nil { + return err + } + if _, err := tw.Write([]byte(file.Body)); err != nil { + return err + } + } + if err := tw.Close(); err != nil { + return err + } + + logger.Debugf("Extracting content to '%s'", dstPath) + err := cr.cli.CopyToContainer(ctx, cr.id, dstPath, &buf, types.CopyToContainerOptions{}) + if err != nil { + return errors.WithStack(err) } return nil } @@ -207,7 +359,7 @@ func (cr *containerReference) attach() common.Executor { func (cr *containerReference) start() common.Executor { return func(ctx context.Context) error { logger := common.Logger(ctx) - logger.Debugf("STARTING image=%s entrypoint=%s cmd=%v", cr.input.Image, cr.input.Entrypoint, cr.input.Cmd) + logger.Debugf("Starting container: %v", cr.id) if err := cr.cli.ContainerStart(ctx, cr.id, types.ContainerStartOptions{}); err != nil { return errors.WithStack(err) diff --git a/pkg/model/workflow.go b/pkg/model/workflow.go index f0c2a462..33cf6e7f 100644 --- a/pkg/model/workflow.go +++ b/pkg/model/workflow.go @@ -3,10 +3,11 @@ package model import ( "fmt" "io" - "log" "regexp" "strings" + "github.com/nektos/act/pkg/common" + log "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" ) @@ -94,6 +95,58 @@ func (j *Job) Needs() []string { return nil } +// GetMatrixes returns the matrix cross product +func (j *Job) GetMatrixes() []map[string]interface{} { + matrixes := make([]map[string]interface{}, 0) + if j.Strategy != nil { + includes := make([]map[string]interface{}, 0) + for _, v := range j.Strategy.Matrix["include"] { + includes = append(includes, v.(map[string]interface{})) + } + delete(j.Strategy.Matrix, "include") + + excludes := make([]map[string]interface{}, 0) + for _, v := range j.Strategy.Matrix["exclude"] { + excludes = append(excludes, v.(map[string]interface{})) + } + delete(j.Strategy.Matrix, "exclude") + + matrixProduct := common.CartesianProduct(j.Strategy.Matrix) + + MATRIX: + for _, matrix := range matrixProduct { + for _, exclude := range excludes { + if commonKeysMatch(matrix, exclude) { + log.Debugf("Skipping matrix '%v' due to exclude '%v'", matrix, exclude) + continue MATRIX + } + } + for _, include := range includes { + if commonKeysMatch(matrix, include) { + log.Debugf("Setting add'l values on matrix '%v' due to include '%v'", matrix, include) + for k, v := range include { + matrix[k] = v + } + } + } + matrixes = append(matrixes, matrix) + } + + } else { + matrixes = append(matrixes, make(map[string]interface{})) + } + return matrixes +} + +func commonKeysMatch(a map[string]interface{}, b map[string]interface{}) bool { + for aKey, aVal := range a { + if bVal, ok := b[aKey]; ok && aVal != bVal { + return false + } + } + return true +} + // ContainerSpec is the specification of the container to use for the job type ContainerSpec struct { Image string `yaml:"image"` @@ -148,6 +201,29 @@ func (s *Step) GetEnv() map[string]string { return rtnEnv } +// ShellCommand returns the command for the shell +func (s *Step) ShellCommand() string { + shellCommand := "" + + switch s.Shell { + case "", "bash": + shellCommand = "bash --noprofile --norc -eo pipefail {0}" + case "pwsh": + shellCommand = "pwsh -command \"& '{0}'\"" + case "python": + shellCommand = "python {0}" + case "sh": + shellCommand = "sh -e -c {0}" + case "cmd": + shellCommand = "%ComSpec% /D /E:ON /V:OFF /S /C \"CALL \"{0}\"\"" + case "powershell": + shellCommand = "powershell -command \"& '{0}'\"" + default: + shellCommand = s.Shell + } + return shellCommand +} + // StepType describes what type of step we are about to run type StepType int diff --git a/pkg/runner/expression.go b/pkg/runner/expression.go index 9226faf1..9de525c5 100644 --- a/pkg/runner/expression.go +++ b/pkg/runner/expression.go @@ -11,7 +11,6 @@ import ( "regexp" "strings" - "github.com/nektos/act/pkg/model" "github.com/robertkrimen/otto" "github.com/sirupsen/logrus" "gopkg.in/godo.v2/glob" @@ -34,11 +33,11 @@ func (rc *RunContext) NewExpressionEvaluator() ExpressionEvaluator { } } -// NewStepExpressionEvaluator creates a new evaluator -func (rc *RunContext) NewStepExpressionEvaluator(step *model.Step) ExpressionEvaluator { - vm := rc.newVM() +// NewExpressionEvaluator creates a new evaluator +func (sc *StepContext) NewExpressionEvaluator() ExpressionEvaluator { + vm := sc.RunContext.newVM() configers := []func(*otto.Otto){ - rc.vmEnv(step), + sc.vmEnv(), } for _, configer := range configers { configer(vm) @@ -236,10 +235,9 @@ func (rc *RunContext) vmGithub() func(*otto.Otto) { } } -func (rc *RunContext) vmEnv(step *model.Step) func(*otto.Otto) { +func (sc *StepContext) vmEnv() func(*otto.Otto) { return func(vm *otto.Otto) { - env := rc.StepEnv(step) - _ = vm.Set("env", env) + _ = vm.Set("env", sc.Env) } } diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go index 5a662d65..39b3737b 100644 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -1,16 +1,12 @@ package runner import ( - "archive/tar" - "bytes" "context" "encoding/json" "fmt" - "io" - "io/ioutil" "os" + "path/filepath" "regexp" - "runtime" "strings" "github.com/nektos/act/pkg/container" @@ -23,16 +19,16 @@ import ( // RunContext contains info about current job type RunContext struct { - Config *Config - Matrix map[string]interface{} - Run *model.Run - EventJSON string - Env map[string]string - Tempdir string - ExtraPath []string - CurrentStep string - StepResults map[string]*stepResult - ExprEval ExpressionEvaluator + Config *Config + Matrix map[string]interface{} + Run *model.Run + EventJSON string + Env map[string]string + ExtraPath []string + CurrentStep string + StepResults map[string]*stepResult + ExprEval ExpressionEvaluator + JobContainer container.Container } type stepResult struct { @@ -48,80 +44,141 @@ func (rc *RunContext) GetEnv() map[string]string { return rc.Env } -// Close cleans up temp dir -func (rc *RunContext) Close(ctx context.Context) error { - return os.RemoveAll(rc.Tempdir) +func (rc *RunContext) jobContainerName() string { + return createContainerName(filepath.Base(rc.Config.Workdir), rc.Run.String()) +} + +func (rc *RunContext) startJobContainer() common.Executor { + job := rc.Run.Job() + + var image string + if job.Container != nil { + image = job.Container.Image + } else { + platformName := rc.ExprEval.Interpolate(job.RunsOn) + image = rc.Config.Platforms[strings.ToLower(platformName)] + } + + return func(ctx context.Context) error { + rawLogger := common.Logger(ctx).WithField("raw_output", true) + logWriter := common.NewLineWriter(rc.commandHandler(ctx), func(s string) { + if rc.Config.LogOutput { + rawLogger.Infof(s) + } else { + rawLogger.Debugf(s) + } + }) + + common.Logger(ctx).Infof("\U0001f680 Start image=%s", image) + name := rc.jobContainerName() + + rc.JobContainer = container.NewContainer(&container.NewContainerInput{ + Cmd: nil, + Entrypoint: []string{"/bin/cat"}, + WorkingDir: "/github/workspace", + Image: image, + Name: name, + Mounts: map[string]string{ + name: "/github", + }, + Binds: []string{ + fmt.Sprintf("%s:%s", rc.Config.Workdir, "/github/workspace"), + fmt.Sprintf("%s:%s", "/var/run/docker.sock", "/var/run/docker.sock"), + }, + Stdout: logWriter, + Stderr: logWriter, + }) + + return common.NewPipelineExecutor( + rc.JobContainer.Pull(rc.Config.ForcePull), + rc.JobContainer.Remove().IfBool(!rc.Config.ReuseContainers), + rc.JobContainer.Create(), + rc.JobContainer.Start(false), + rc.JobContainer.Copy("/github/", &container.FileEntry{ + Name: "workflow/event.json", + Mode: 644, + Body: rc.EventJSON, + }), + )(ctx) + } +} +func (rc *RunContext) execJobContainer(cmd []string, env map[string]string) common.Executor { + return func(ctx context.Context) error { + return rc.JobContainer.Exec(cmd, env)(ctx) + } +} +func (rc *RunContext) stopJobContainer() common.Executor { + return func(ctx context.Context) error { + if rc.JobContainer != nil && !rc.Config.ReuseContainers { + return rc.JobContainer.Remove(). + Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName(), false))(ctx) + } + return nil + } } // Executor returns a pipeline executor for all the steps in the job func (rc *RunContext) Executor() common.Executor { - - err := rc.setupTempDir() - if err != nil { - return common.NewErrorExecutor(err) - } steps := make([]common.Executor, 0) + steps = append(steps, rc.startJobContainer()) for i, step := range rc.Run.Job().Steps { if step.ID == "" { step.ID = fmt.Sprintf("%d", i) } - s := step - steps = append(steps, func(ctx context.Context) error { - rc.CurrentStep = s.ID - rc.StepResults[rc.CurrentStep] = &stepResult{ - Success: true, - Outputs: make(map[string]string), - } - rc.ExprEval = rc.NewStepExpressionEvaluator(s) + steps = append(steps, rc.newStepExecutor(step)) + } + steps = append(steps, rc.stopJobContainer()) - if !rc.EvalBool(s.If) { - log.Debugf("Skipping step '%s' due to '%s'", s.String(), s.If) - return nil - } + return common.NewPipelineExecutor(steps...).If(rc.isEnabled) +} - common.Logger(ctx).Infof("\u2B50 Run %s", s) - err := rc.newStepExecutor(s)(ctx) - if err == nil { - common.Logger(ctx).Infof(" \u2705 Success - %s", s) - } else { - common.Logger(ctx).Errorf(" \u274C Failure - %s", s) - rc.StepResults[rc.CurrentStep].Success = false - } - return err - }) +func (rc *RunContext) newStepExecutor(step *model.Step) common.Executor { + sc := &StepContext{ + RunContext: rc, + Step: step, } return func(ctx context.Context) error { - defer rc.Close(ctx) - job := rc.Run.Job() - log := common.Logger(ctx) - if !rc.EvalBool(job.If) { - log.Debugf("Skipping job '%s' due to '%s'", job.Name, job.If) + rc.CurrentStep = sc.Step.ID + rc.StepResults[rc.CurrentStep] = &stepResult{ + Success: true, + Outputs: make(map[string]string), + } + rc.ExprEval = sc.NewExpressionEvaluator() + + if !rc.EvalBool(sc.Step.If) { + log.Debugf("Skipping step '%s' due to '%s'", sc.Step.String(), sc.Step.If) return nil } - platformName := rc.ExprEval.Interpolate(rc.Run.Job().RunsOn) - if img, ok := rc.Config.Platforms[strings.ToLower(platformName)]; !ok || img == "" { - log.Infof(" \U0001F6A7 Skipping unsupported platform '%s'", platformName) - return nil + common.Logger(ctx).Infof("\u2B50 Run %s", sc.Step) + err := sc.Executor()(ctx) + if err == nil { + common.Logger(ctx).Infof(" \u2705 Success - %s", sc.Step) + } else { + common.Logger(ctx).Errorf(" \u274C Failure - %s", sc.Step) + rc.StepResults[rc.CurrentStep].Success = false } - - nullLogger := logrus.New() - nullLogger.Out = ioutil.Discard - if !rc.Config.ReuseContainers { - _ = rc.newContainerCleaner()(common.WithLogger(ctx, nullLogger)) - } - - err := common.NewPipelineExecutor(steps...)(ctx) - - if !rc.Config.ReuseContainers { - _ = rc.newContainerCleaner()(common.WithLogger(ctx, nullLogger)) - } - return err } } +func (rc *RunContext) isEnabled(ctx context.Context) bool { + job := rc.Run.Job() + log := common.Logger(ctx) + if !rc.EvalBool(job.If) { + log.Debugf("Skipping job '%s' due to '%s'", job.Name, job.If) + return false + } + + platformName := rc.ExprEval.Interpolate(rc.Run.Job().RunsOn) + if img, ok := rc.Config.Platforms[strings.ToLower(platformName)]; !ok || img == "" { + log.Infof(" \U0001F6A7 Skipping unsupported platform '%s'", platformName) + return false + } + return true +} + // EvalBool evaluates an expression against current run context func (rc *RunContext) EvalBool(expr string) bool { if expr != "" { @@ -145,33 +202,7 @@ func mergeMaps(maps ...map[string]string) map[string]string { return rtnMap } -func (rc *RunContext) setupTempDir() error { - var err error - tempBase := "" - if runtime.GOOS == "darwin" { - tempBase = "/tmp" - } - rc.Tempdir, err = ioutil.TempDir(tempBase, "act-") - if err != nil { - return err - } - err = os.Chmod(rc.Tempdir, 0755) - if err != nil { - return err - } - log.Debugf("Setup tempdir %s", rc.Tempdir) - return err -} - -func (rc *RunContext) pullImage(containerSpec *model.ContainerSpec) common.Executor { - return func(ctx context.Context) error { - return container.NewDockerPullExecutor(container.NewDockerPullExecutorInput{ - Image: containerSpec.Image, - ForcePull: rc.Config.ForcePull, - })(ctx) - } -} - +/* func (rc *RunContext) runContainer(containerSpec *model.ContainerSpec) common.Executor { return func(ctx context.Context) error { ghReader, err := rc.createGithubTarball() @@ -200,7 +231,7 @@ func (rc *RunContext) runContainer(containerSpec *model.ContainerSpec) common.Ex } }) - return container.NewDockerRunExecutor(container.NewDockerRunExecutorInput{ + c := container.NewContainer(&container.NewContainerInput{ Cmd: cmd, Entrypoint: entrypoint, Image: containerSpec.Image, @@ -212,64 +243,27 @@ func (rc *RunContext) runContainer(containerSpec *model.ContainerSpec) common.Ex fmt.Sprintf("%s:%s", rc.Tempdir, "/github/home"), fmt.Sprintf("%s:%s", "/var/run/docker.sock", "/var/run/docker.sock"), }, - Content: map[string]io.Reader{"/github": ghReader}, - ReuseContainers: containerSpec.Reuse, - Stdout: logWriter, - Stderr: logWriter, - })(ctx) + Stdout: logWriter, + Stderr: logWriter, + }) + + return c.Create(). + Then(c.Copy("/github", ghReader)). + Then(c.Start()). + Finally(c.Remove().IfBool(!rc.Config.ReuseContainers))(ctx) + } } -func (rc *RunContext) createGithubTarball() (io.Reader, error) { - var buf bytes.Buffer - tw := tar.NewWriter(&buf) - var files = []struct { - Name string - Mode int64 - Body string - }{ - {"workflow/event.json", 0644, rc.EventJSON}, +*/ + +func createContainerName(parts ...string) string { + name := make([]string, 0) + pattern := regexp.MustCompile("[^a-zA-Z0-9]") + for _, part := range parts { + name = append(name, pattern.ReplaceAllString(part, "-")) } - for _, file := range files { - log.Debugf("Writing entry to tarball %s len:%d", file.Name, len(rc.EventJSON)) - hdr := &tar.Header{ - Name: file.Name, - Mode: file.Mode, - Size: int64(len(rc.EventJSON)), - } - if err := tw.WriteHeader(hdr); err != nil { - return nil, err - } - if _, err := tw.Write([]byte(rc.EventJSON)); err != nil { - return nil, err - } - } - if err := tw.Close(); err != nil { - return nil, err - } - - return &buf, nil - -} - -func (rc *RunContext) createContainerName() string { - containerName := rc.Run.String() - containerName = regexp.MustCompile("[^a-zA-Z0-9]").ReplaceAllString(containerName, "-") - - prefix := "" - suffix := "" - containerName = trimToLen(containerName, 30-(len(prefix)+len(suffix))) - return fmt.Sprintf("%s%s%s", prefix, containerName, suffix) - -} - -func (rc *RunContext) createStepContainerName(stepID string) string { - - prefix := regexp.MustCompile("[^a-zA-Z0-9]").ReplaceAllString(rc.createContainerName(), "-") - suffix := regexp.MustCompile("[^a-zA-Z0-9]").ReplaceAllString(stepID, "-") - prefix = trimToLen(prefix, 30-(1+len(suffix))) - name := strings.Trim(fmt.Sprintf("%s-%s", prefix, suffix), "-") - return name + return trimToLen(strings.Join(name, "-"), 30) } func trimToLen(s string, l int) string { diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index 2c1888a4..6aa1eaf7 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -13,7 +13,6 @@ import ( // Runner provides capabilities to run GitHub actions type Runner interface { NewPlanExecutor(plan *model.Plan) common.Executor - NewRunExecutor(run *model.Run, matrix map[string]interface{}) common.Executor } // Config contains the config for a new runner @@ -59,49 +58,12 @@ func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor { stageExecutor := make([]common.Executor, 0) for _, run := range stage.Runs { job := run.Job() - matrixes := make([]map[string]interface{}, 0) - if job.Strategy != nil { - includes := make([]map[string]interface{}, 0) - for _, v := range job.Strategy.Matrix["include"] { - includes = append(includes, v.(map[string]interface{})) - } - delete(job.Strategy.Matrix, "include") - - excludes := make([]map[string]interface{}, 0) - for _, v := range job.Strategy.Matrix["exclude"] { - excludes = append(excludes, v.(map[string]interface{})) - } - delete(job.Strategy.Matrix, "exclude") - - matrixProduct := common.CartesianProduct(job.Strategy.Matrix) - - MATRIX: - for _, matrix := range matrixProduct { - for _, exclude := range excludes { - if commonKeysMatch(matrix, exclude) { - log.Debugf("Skipping matrix '%v' due to exclude '%v'", matrix, exclude) - continue MATRIX - } - } - for _, include := range includes { - if commonKeysMatch(matrix, include) { - log.Debugf("Setting add'l values on matrix '%v' due to include '%v'", matrix, include) - for k, v := range include { - matrix[k] = v - } - } - } - matrixes = append(matrixes, matrix) - } - - } else { - matrixes = append(matrixes, make(map[string]interface{})) - } + matrixes := job.GetMatrixes() jobName := fmt.Sprintf("%-*s", maxJobNameLen, run.String()) for _, matrix := range matrixes { m := matrix - runExecutor := runner.NewRunExecutor(run, matrix) + runExecutor := runner.newRunExecutor(run, matrix) stageExecutor = append(stageExecutor, func(ctx context.Context) error { ctx = WithJobLogger(ctx, jobName) if len(m) > 0 { @@ -117,22 +79,14 @@ func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor { return common.NewPipelineExecutor(pipeline...) } -func commonKeysMatch(a map[string]interface{}, b map[string]interface{}) bool { - for aKey, aVal := range a { - if bVal, ok := b[aKey]; ok && aVal != bVal { - return false - } +func (runner *runnerImpl) newRunExecutor(run *model.Run, matrix map[string]interface{}) common.Executor { + rc := &RunContext{ + Config: runner.config, + Run: run, + EventJSON: runner.eventJSON, + StepResults: make(map[string]*stepResult), + Matrix: matrix, } - return true -} - -func (runner *runnerImpl) NewRunExecutor(run *model.Run, matrix map[string]interface{}) common.Executor { - rc := new(RunContext) - rc.Config = runner.config - rc.Run = run - rc.EventJSON = runner.eventJSON - rc.StepResults = make(map[string]*stepResult) - rc.Matrix = matrix rc.ExprEval = rc.NewExpressionEvaluator() return rc.Executor() } diff --git a/pkg/runner/step.go b/pkg/runner/step.go deleted file mode 100644 index e5cf61c4..00000000 --- a/pkg/runner/step.go +++ /dev/null @@ -1,276 +0,0 @@ -package runner - -import ( - "context" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "regexp" - "strings" - - "github.com/nektos/act/pkg/common" - "github.com/nektos/act/pkg/container" - "github.com/nektos/act/pkg/model" - log "github.com/sirupsen/logrus" -) - -func (rc *RunContext) StepEnv(step *model.Step) map[string]string { - var env map[string]string - job := rc.Run.Job() - if job.Container != nil { - env = mergeMaps(rc.GetEnv(), job.Container.Env, step.GetEnv()) - } else { - env = mergeMaps(rc.GetEnv(), step.GetEnv()) - } - - for k, v := range env { - env[k] = rc.ExprEval.Interpolate(v) - } - return env -} - -func (rc *RunContext) setupEnv(containerSpec *model.ContainerSpec, step *model.Step) common.Executor { - return func(ctx context.Context) error { - containerSpec.Env = rc.withGithubEnv(rc.StepEnv(step)) - return nil - } -} - -func (rc *RunContext) newContainerCleaner() common.Executor { - job := rc.Run.Job() - containerSpec := new(model.ContainerSpec) - containerSpec.Name = rc.createContainerName() - containerSpec.Reuse = false - - if job.Container != nil { - containerSpec.Image = job.Container.Image - } else { - platformName := rc.ExprEval.Interpolate(rc.Run.Job().RunsOn) - containerSpec.Image = rc.Config.Platforms[strings.ToLower(platformName)] - } - containerSpec.Entrypoint = "bash --noprofile --norc -o pipefail -c echo 'cleaning up'" - return common.NewPipelineExecutor( - rc.pullImage(containerSpec), - rc.runContainer(containerSpec), - ) -} - -func (rc *RunContext) newStepExecutor(step *model.Step) common.Executor { - job := rc.Run.Job() - containerSpec := new(model.ContainerSpec) - containerSpec.Name = rc.createContainerName() - containerSpec.Reuse = true - - if job.Container != nil { - containerSpec.Image = job.Container.Image - } else { - platformName := rc.ExprEval.Interpolate(rc.Run.Job().RunsOn) - containerSpec.Image = rc.Config.Platforms[strings.ToLower(platformName)] - } - - switch step.Type() { - case model.StepTypeRun: - if job.Container != nil { - containerSpec.Ports = job.Container.Ports - containerSpec.Volumes = job.Container.Volumes - containerSpec.Options = job.Container.Options - } - return common.NewPipelineExecutor( - rc.setupEnv(containerSpec, step), - rc.setupShellCommand(containerSpec, step.Shell, step.Run), - rc.pullImage(containerSpec), - rc.runContainer(containerSpec), - ) - - case model.StepTypeUsesDockerURL: - containerSpec.Image = strings.TrimPrefix(step.Uses, "docker://") - containerSpec.Name = rc.createStepContainerName(step.ID) - containerSpec.Entrypoint = step.With["entrypoint"] - containerSpec.Args = step.With["args"] - containerSpec.Reuse = rc.Config.ReuseContainers - return common.NewPipelineExecutor( - rc.setupEnv(containerSpec, step), - rc.pullImage(containerSpec), - rc.runContainer(containerSpec), - ) - - case model.StepTypeUsesActionLocal: - return common.NewPipelineExecutor( - rc.setupEnv(containerSpec, step), - rc.setupAction(containerSpec, filepath.Join(rc.Config.Workdir, step.Uses)), - applyWith(containerSpec, step), - rc.pullImage(containerSpec), - rc.runContainer(containerSpec), - ) - case model.StepTypeUsesActionRemote: - remoteAction := newRemoteAction(step.Uses) - if remoteAction.Org == "actions" && remoteAction.Repo == "checkout" { - return func(ctx context.Context) error { - common.Logger(ctx).Debugf("Skipping actions/checkout") - return nil - } - } - cloneDir, err := ioutil.TempDir(rc.Tempdir, remoteAction.Repo) - if err != nil { - return common.NewErrorExecutor(err) - } - return common.NewPipelineExecutor( - common.NewGitCloneExecutor(common.NewGitCloneExecutorInput{ - URL: remoteAction.CloneURL(), - Ref: remoteAction.Ref, - Dir: cloneDir, - }), - rc.setupEnv(containerSpec, step), - rc.setupAction(containerSpec, filepath.Join(cloneDir, remoteAction.Path)), - applyWith(containerSpec, step), - rc.pullImage(containerSpec), - rc.runContainer(containerSpec), - ) - } - - return common.NewErrorExecutor(fmt.Errorf("Unable to determine how to run job:%s step:%+v", rc.Run, step)) -} - -func applyWith(containerSpec *model.ContainerSpec, step *model.Step) common.Executor { - return func(ctx context.Context) error { - if entrypoint, ok := step.With["entrypoint"]; ok { - containerSpec.Entrypoint = entrypoint - } - if args, ok := step.With["args"]; ok { - containerSpec.Args = args - } - return nil - } -} - -func (rc *RunContext) setupShellCommand(containerSpec *model.ContainerSpec, shell string, run string) common.Executor { - return func(ctx context.Context) error { - shellCommand := "" - - switch shell { - case "", "bash": - shellCommand = "bash --noprofile --norc -eo pipefail {0}" - case "pwsh": - shellCommand = "pwsh -command \"& '{0}'\"" - case "python": - shellCommand = "python {0}" - case "sh": - shellCommand = "sh -e -c {0}" - case "cmd": - shellCommand = "%ComSpec% /D /E:ON /V:OFF /S /C \"CALL \"{0}\"\"" - case "powershell": - shellCommand = "powershell -command \"& '{0}'\"" - default: - shellCommand = shell - } - - tempScript, err := ioutil.TempFile(rc.Tempdir, ".temp-script-") - if err != nil { - return err - } - - _, err = tempScript.WriteString(fmt.Sprintf("PATH=\"%s:${PATH}\"\n", strings.Join(rc.ExtraPath, ":"))) - if err != nil { - return err - } - - run = rc.ExprEval.Interpolate(run) - - if _, err := tempScript.WriteString(run); err != nil { - return err - } - log.Debugf("Wrote command '%s' to '%s'", run, tempScript.Name()) - if err := tempScript.Close(); err != nil { - return err - } - containerPath := fmt.Sprintf("/github/home/%s", filepath.Base(tempScript.Name())) - containerSpec.Entrypoint = strings.Replace(shellCommand, "{0}", containerPath, 1) - return nil - } -} - -func (rc *RunContext) setupAction(containerSpec *model.ContainerSpec, actionDir string) common.Executor { - return func(ctx context.Context) error { - f, err := os.Open(filepath.Join(actionDir, "action.yml")) - if os.IsNotExist(err) { - f, err = os.Open(filepath.Join(actionDir, "action.yaml")) - if err != nil { - return err - } - } else if err != nil { - return err - } - - action, err := model.ReadAction(f) - if err != nil { - return err - } - - for inputID, input := range action.Inputs { - envKey := regexp.MustCompile("[^A-Z0-9-]").ReplaceAllString(strings.ToUpper(inputID), "_") - envKey = fmt.Sprintf("INPUT_%s", envKey) - if _, ok := containerSpec.Env[envKey]; !ok { - containerSpec.Env[envKey] = input.Default - } - } - - switch action.Runs.Using { - case model.ActionRunsUsingNode12: - if strings.HasPrefix(actionDir, rc.Config.Workdir) { - containerSpec.Entrypoint = fmt.Sprintf("node /github/workspace/%s/%s", strings.TrimPrefix(actionDir, rc.Config.Workdir), action.Runs.Main) - } else if strings.HasPrefix(actionDir, rc.Tempdir) { - containerSpec.Entrypoint = fmt.Sprintf("node /github/home/%s/%s", strings.TrimPrefix(actionDir, rc.Tempdir), action.Runs.Main) - } - case model.ActionRunsUsingDocker: - if strings.HasPrefix(actionDir, rc.Config.Workdir) { - containerSpec.Name = rc.createStepContainerName(strings.TrimPrefix(actionDir, rc.Config.Workdir)) - } else if strings.HasPrefix(actionDir, rc.Tempdir) { - containerSpec.Name = rc.createStepContainerName(strings.TrimPrefix(actionDir, rc.Tempdir)) - } - containerSpec.Reuse = rc.Config.ReuseContainers - if strings.HasPrefix(action.Runs.Image, "docker://") { - containerSpec.Image = strings.TrimPrefix(action.Runs.Image, "docker://") - containerSpec.Entrypoint = strings.Join(action.Runs.Entrypoint, " ") - containerSpec.Args = strings.Join(action.Runs.Args, " ") - } else { - containerSpec.Image = fmt.Sprintf("%s:%s", containerSpec.Name, "latest") - contextDir := filepath.Join(actionDir, action.Runs.Main) - return container.NewDockerBuildExecutor(container.NewDockerBuildExecutorInput{ - ContextDir: contextDir, - ImageTag: containerSpec.Image, - })(ctx) - } - } - return nil - } -} - -type remoteAction struct { - Org string - Repo string - Path string - Ref string -} - -func (ra *remoteAction) CloneURL() string { - return fmt.Sprintf("https://github.com/%s/%s", ra.Org, ra.Repo) -} - -func newRemoteAction(action string) *remoteAction { - r := regexp.MustCompile(`^([^/@]+)/([^/@]+)(/([^@]*))?(@(.*))?$`) - matches := r.FindStringSubmatch(action) - - ra := new(remoteAction) - ra.Org = matches[1] - ra.Repo = matches[2] - ra.Path = "" - ra.Ref = "master" - if len(matches) >= 5 { - ra.Path = matches[4] - } - if len(matches) >= 7 { - ra.Ref = matches[6] - } - return ra -} diff --git a/pkg/runner/testdata/basic/push.yml b/pkg/runner/testdata/basic/push.yml index 720ca259..50c6553a 100644 --- a/pkg/runner/testdata/basic/push.yml +++ b/pkg/runner/testdata/basic/push.yml @@ -5,9 +5,12 @@ jobs: check: runs-on: ubuntu-latest steps: - - run: echo 'hello world' + - run: echo 'hello world' + - run: echo ${GITHUB_SHA} >> /github/sha.txt + - run: cat /github/sha.txt | grep ${GITHUB_SHA} build: + if: false runs-on: ubuntu-latest needs: [check] steps: @@ -20,4 +23,8 @@ jobs: steps: - uses: docker://ubuntu:18.04 with: - args: echo ${GITHUB_REF} | grep nektos/act + args: env + - uses: docker://ubuntu:18.04 + with: + entrypoint: /bin/echo + args: ${{github.event_name}} diff --git a/pkg/runner/testdata/node/push.yml b/pkg/runner/testdata/node/push.yml index 0eebcf42..e8a284d3 100644 --- a/pkg/runner/testdata/node/push.yml +++ b/pkg/runner/testdata/node/push.yml @@ -6,7 +6,6 @@ jobs: test: runs-on: ubuntu-latest steps: - - run: which node - uses: actions/checkout@v2 - uses: actions/setup-node@v1 with: From ac1bd0893eb18724138c0d97b41327ce1ce6ba1c Mon Sep 17 00:00:00 2001 From: Casey Lee Date: Sun, 23 Feb 2020 15:02:01 -0800 Subject: [PATCH 05/14] shared container for job --- pkg/container/docker_volume.go | 29 ++++ pkg/runner/step_context.go | 290 +++++++++++++++++++++++++++++++++ 2 files changed, 319 insertions(+) create mode 100644 pkg/container/docker_volume.go create mode 100644 pkg/runner/step_context.go diff --git a/pkg/container/docker_volume.go b/pkg/container/docker_volume.go new file mode 100644 index 00000000..a0b533f5 --- /dev/null +++ b/pkg/container/docker_volume.go @@ -0,0 +1,29 @@ +package container + +import ( + "context" + + "github.com/docker/docker/client" + "github.com/nektos/act/pkg/common" +) + +// NewDockerVolumeRemoveExecutor function +func NewDockerVolumeRemoveExecutor(volume string, force bool) common.Executor { + return func(ctx context.Context) error { + logger := common.Logger(ctx) + logger.Debugf("%sdocker volume rm %s", logPrefix, volume) + + if common.Dryrun(ctx) { + return nil + } + + cli, err := client.NewClientWithOpts(client.FromEnv) + if err != nil { + return err + } + cli.NegotiateAPIVersion(ctx) + + return cli.VolumeRemove(ctx, volume, force) + } + +} diff --git a/pkg/runner/step_context.go b/pkg/runner/step_context.go new file mode 100644 index 00000000..0e4c493f --- /dev/null +++ b/pkg/runner/step_context.go @@ -0,0 +1,290 @@ +package runner + +import ( + "context" + "fmt" + "regexp" + "strings" + + "github.com/nektos/act/pkg/common" + "github.com/nektos/act/pkg/container" + "github.com/nektos/act/pkg/model" + log "github.com/sirupsen/logrus" +) + +// StepContext contains info about current job +type StepContext struct { + RunContext *RunContext + Step *model.Step + Env map[string]string + Cmd []string +} + +func (sc *StepContext) execJobContainer() common.Executor { + return func(ctx context.Context) error { + return sc.RunContext.execJobContainer(sc.Cmd, sc.Env)(ctx) + } +} + +// Executor for a step context +func (sc *StepContext) Executor() common.Executor { + rc := sc.RunContext + step := sc.Step + + switch step.Type() { + case model.StepTypeRun: + return common.NewPipelineExecutor( + sc.setupEnv(), + sc.setupShellCommand(), + sc.execJobContainer(), + ) + + case model.StepTypeUsesDockerURL: + return common.NewPipelineExecutor( + sc.setupEnv(), + sc.runUsesContainer(), + ) + + /* + case model.StepTypeUsesActionLocal: + return common.NewPipelineExecutor( + sc.setupEnv(), + sc.setupAction(), + applyWith(containerSpec, step), + rc.pullImage(containerSpec), + rc.runContainer(containerSpec), + ) + case model.StepTypeUsesActionRemote: + remoteAction := newRemoteAction(step.Uses) + if remoteAction.Org == "actions" && remoteAction.Repo == "checkout" { + return func(ctx context.Context) error { + common.Logger(ctx).Debugf("Skipping actions/checkout") + return nil + } + } + cloneDir, err := ioutil.TempDir(rc.Tempdir, remoteAction.Repo) + if err != nil { + return common.NewErrorExecutor(err) + } + return common.NewPipelineExecutor( + common.NewGitCloneExecutor(common.NewGitCloneExecutorInput{ + URL: remoteAction.CloneURL(), + Ref: remoteAction.Ref, + Dir: cloneDir, + }), + sc.setupEnv(), + sc.setupAction(), + applyWith(containerSpec, step), + rc.pullImage(containerSpec), + rc.runContainer(containerSpec), + ) + */ + } + + return common.NewErrorExecutor(fmt.Errorf("Unable to determine how to run job:%s step:%+v", rc.Run, step)) +} + +func applyWith(containerSpec *model.ContainerSpec, step *model.Step) common.Executor { + return func(ctx context.Context) error { + if entrypoint, ok := step.With["entrypoint"]; ok { + containerSpec.Entrypoint = entrypoint + } + if args, ok := step.With["args"]; ok { + containerSpec.Args = args + } + return nil + } +} + +func (sc *StepContext) setupEnv() common.Executor { + rc := sc.RunContext + job := rc.Run.Job() + step := sc.Step + return func(ctx context.Context) error { + var env map[string]string + if job.Container != nil { + env = mergeMaps(rc.GetEnv(), job.Container.Env, step.GetEnv()) + } else { + env = mergeMaps(rc.GetEnv(), step.GetEnv()) + } + + for k, v := range env { + env[k] = rc.ExprEval.Interpolate(v) + } + sc.Env = rc.withGithubEnv(env) + return nil + } +} + +func (sc *StepContext) setupShellCommand() common.Executor { + rc := sc.RunContext + step := sc.Step + return func(ctx context.Context) error { + var script strings.Builder + + _, err := script.WriteString(fmt.Sprintf("PATH=\"%s:${PATH}\"\n", strings.Join(rc.ExtraPath, ":"))) + if err != nil { + return err + } + + run := rc.ExprEval.Interpolate(step.Run) + + if _, err = script.WriteString(run); err != nil { + return err + } + scriptName := fmt.Sprintf("workflow/%s", step.ID) + log.Debugf("Wrote command '%s' to '%s'", run, scriptName) + containerPath := fmt.Sprintf("/github/%s", scriptName) + sc.Cmd = strings.Fields(strings.Replace(step.ShellCommand(), "{0}", containerPath, 1)) + return rc.JobContainer.Copy("/github/", &container.FileEntry{ + Name: scriptName, + Mode: 755, + Body: script.String(), + })(ctx) + } +} + +func (sc *StepContext) newStepContainer(ctx context.Context, image string, cmd []string, entrypoint []string) container.Container { + rc := sc.RunContext + step := sc.Step + rawLogger := common.Logger(ctx).WithField("raw_output", true) + logWriter := common.NewLineWriter(rc.commandHandler(ctx), func(s string) { + if rc.Config.LogOutput { + rawLogger.Infof(s) + } else { + rawLogger.Debugf(s) + } + }) + envList := make([]string, 0) + for k, v := range sc.Env { + envList = append(envList, fmt.Sprintf("%s=%s", k, v)) + } + stepContainer := container.NewContainer(&container.NewContainerInput{ + Cmd: cmd, + Entrypoint: entrypoint, + WorkingDir: "/github/workspace", + Image: image, + Name: createContainerName(rc.jobContainerName(), step.ID), + Env: envList, + Mounts: map[string]string{ + rc.jobContainerName(): "/github", + }, + Binds: []string{ + fmt.Sprintf("%s:%s", rc.Config.Workdir, "/github/workspace"), + fmt.Sprintf("%s:%s", "/var/run/docker.sock", "/var/run/docker.sock"), + }, + Stdout: logWriter, + Stderr: logWriter, + }) + return stepContainer +} +func (sc *StepContext) runUsesContainer() common.Executor { + rc := sc.RunContext + step := sc.Step + return func(ctx context.Context) error { + image := strings.TrimPrefix(step.Uses, "docker://") + cmd := strings.Fields(rc.ExprEval.Interpolate(step.With["args"])) + entrypoint := strings.Fields(rc.ExprEval.Interpolate(step.With["entrypoint"])) + stepContainer := sc.newStepContainer(ctx, image, cmd, entrypoint) + + return common.NewPipelineExecutor( + stepContainer.Pull(rc.Config.ForcePull), + stepContainer.Remove().IfBool(!rc.Config.ReuseContainers), + stepContainer.Create(), + stepContainer.Start(true), + ).Finally( + stepContainer.Remove().IfBool(!rc.Config.ReuseContainers), + )(ctx) + } +} + +/* + +func (sc *StepContext) setupAction() common.Executor { + rc := sc.RunContext + step := sc.Step + actionDir := filepath.Join(rc.Config.Workdir, step.Uses) + return func(ctx context.Context) error { + f, err := os.Open(filepath.Join(actionDir, "action.yml")) + if os.IsNotExist(err) { + f, err = os.Open(filepath.Join(actionDir, "action.yaml")) + if err != nil { + return err + } + } else if err != nil { + return err + } + + action, err := model.ReadAction(f) + if err != nil { + return err + } + + for inputID, input := range action.Inputs { + envKey := regexp.MustCompile("[^A-Z0-9-]").ReplaceAllString(strings.ToUpper(inputID), "_") + envKey = fmt.Sprintf("INPUT_%s", envKey) + if _, ok := containerSpec.Env[envKey]; !ok { + containerSpec.Env[envKey] = input.Default + } + } + + switch action.Runs.Using { + case model.ActionRunsUsingNode12: + if strings.HasPrefix(actionDir, rc.Config.Workdir) { + containerSpec.Entrypoint = fmt.Sprintf("node /github/workspace/%s/%s", strings.TrimPrefix(actionDir, rc.Config.Workdir), action.Runs.Main) + } else if strings.HasPrefix(actionDir, rc.Tempdir) { + containerSpec.Entrypoint = fmt.Sprintf("node /github/home/%s/%s", strings.TrimPrefix(actionDir, rc.Tempdir), action.Runs.Main) + } + case model.ActionRunsUsingDocker: + if strings.HasPrefix(actionDir, rc.Config.Workdir) { + containerSpec.Name = rc.createStepContainerName(strings.TrimPrefix(actionDir, rc.Config.Workdir)) + } else if strings.HasPrefix(actionDir, rc.Tempdir) { + containerSpec.Name = rc.createStepContainerName(strings.TrimPrefix(actionDir, rc.Tempdir)) + } + containerSpec.Reuse = rc.Config.ReuseContainers + if strings.HasPrefix(action.Runs.Image, "docker://") { + containerSpec.Image = strings.TrimPrefix(action.Runs.Image, "docker://") + containerSpec.Entrypoint = strings.Join(action.Runs.Entrypoint, " ") + containerSpec.Args = strings.Join(action.Runs.Args, " ") + } else { + containerSpec.Image = fmt.Sprintf("%s:%s", containerSpec.Name, "latest") + contextDir := filepath.Join(actionDir, action.Runs.Main) + return container.NewDockerBuildExecutor(container.NewDockerBuildExecutorInput{ + ContextDir: contextDir, + ImageTag: containerSpec.Image, + })(ctx) + } + } + return nil + } +} +*/ + +type remoteAction struct { + Org string + Repo string + Path string + Ref string +} + +func (ra *remoteAction) CloneURL() string { + return fmt.Sprintf("https://github.com/%s/%s", ra.Org, ra.Repo) +} + +func newRemoteAction(action string) *remoteAction { + r := regexp.MustCompile(`^([^/@]+)/([^/@]+)(/([^@]*))?(@(.*))?$`) + matches := r.FindStringSubmatch(action) + + ra := new(remoteAction) + ra.Org = matches[1] + ra.Repo = matches[2] + ra.Path = "" + ra.Ref = "master" + if len(matches) >= 5 { + ra.Path = matches[4] + } + if len(matches) >= 7 { + ra.Ref = matches[6] + } + return ra +} From 94591c58d7d233f6ab0494b1aabc4990d7e69925 Mon Sep 17 00:00:00 2001 From: Casey Lee Date: Sun, 23 Feb 2020 16:36:44 -0800 Subject: [PATCH 06/14] local actions done --- pkg/container/docker_run.go | 4 +- pkg/runner/run_context.go | 64 ++----------- pkg/runner/step_context.go | 149 +++++++++++++++++++---------- pkg/runner/testdata/basic/push.yml | 1 - 4 files changed, 105 insertions(+), 113 deletions(-) diff --git a/pkg/container/docker_run.go b/pkg/container/docker_run.go index 63d83923..81a6464a 100644 --- a/pkg/container/docker_run.go +++ b/pkg/container/docker_run.go @@ -70,7 +70,7 @@ func (cr *containerReference) Create() common.Executor { } func (cr *containerReference) Start(attach bool) common.Executor { return common. - NewDebugExecutor("%sdocker run image=%s entrypoint=%+q cmd=%+q", logPrefix, cr.input.Image, cr.input.Entrypoint, cr.input.Cmd). + NewInfoExecutor("%sdocker run image=%s entrypoint=%+q cmd=%+q", logPrefix, cr.input.Image, cr.input.Entrypoint, cr.input.Cmd). Then( common.NewPipelineExecutor( cr.connect(), @@ -173,9 +173,9 @@ func (cr *containerReference) remove() common.Executor { if err != nil { return errors.WithStack(err) } - cr.id = "" logger.Debugf("Removed container: %v", cr.id) + cr.id = "" return nil } } diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go index 39b3737b..81737250 100644 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -98,6 +98,10 @@ func (rc *RunContext) startJobContainer() common.Executor { Name: "workflow/event.json", Mode: 644, Body: rc.EventJSON, + }, &container.FileEntry{ + Name: "home/.actions/.keep", + Mode: 644, + Body: "", }), )(ctx) } @@ -202,68 +206,14 @@ func mergeMaps(maps ...map[string]string) map[string]string { return rtnMap } -/* -func (rc *RunContext) runContainer(containerSpec *model.ContainerSpec) common.Executor { - return func(ctx context.Context) error { - ghReader, err := rc.createGithubTarball() - if err != nil { - return err - } - - envList := make([]string, 0) - for k, v := range containerSpec.Env { - envList = append(envList, fmt.Sprintf("%s=%s", k, v)) - } - var cmd, entrypoint []string - if containerSpec.Args != "" { - cmd = strings.Fields(rc.ExprEval.Interpolate(containerSpec.Args)) - } - if containerSpec.Entrypoint != "" { - entrypoint = strings.Fields(rc.ExprEval.Interpolate(containerSpec.Entrypoint)) - } - - rawLogger := common.Logger(ctx).WithField("raw_output", true) - logWriter := common.NewLineWriter(rc.commandHandler(ctx), func(s string) { - if rc.Config.LogOutput { - rawLogger.Infof(s) - } else { - rawLogger.Debugf(s) - } - }) - - c := container.NewContainer(&container.NewContainerInput{ - Cmd: cmd, - Entrypoint: entrypoint, - Image: containerSpec.Image, - WorkingDir: "/github/workspace", - Env: envList, - Name: containerSpec.Name, - Binds: []string{ - fmt.Sprintf("%s:%s", rc.Config.Workdir, "/github/workspace"), - fmt.Sprintf("%s:%s", rc.Tempdir, "/github/home"), - fmt.Sprintf("%s:%s", "/var/run/docker.sock", "/var/run/docker.sock"), - }, - Stdout: logWriter, - Stderr: logWriter, - }) - - return c.Create(). - Then(c.Copy("/github", ghReader)). - Then(c.Start()). - Finally(c.Remove().IfBool(!rc.Config.ReuseContainers))(ctx) - - } -} - -*/ - func createContainerName(parts ...string) string { name := make([]string, 0) pattern := regexp.MustCompile("[^a-zA-Z0-9]") + partLen := (30 / len(parts)) - 1 for _, part := range parts { - name = append(name, pattern.ReplaceAllString(part, "-")) + name = append(name, trimToLen(pattern.ReplaceAllString(part, "-"), partLen)) } - return trimToLen(strings.Join(name, "-"), 30) + return strings.Join(name, "-") } func trimToLen(s string, l int) string { diff --git a/pkg/runner/step_context.go b/pkg/runner/step_context.go index 0e4c493f..2a252d00 100644 --- a/pkg/runner/step_context.go +++ b/pkg/runner/step_context.go @@ -3,6 +3,8 @@ package runner import ( "context" "fmt" + "os" + "path/filepath" "regexp" "strings" @@ -18,6 +20,7 @@ type StepContext struct { Step *model.Step Env map[string]string Cmd []string + Action *model.Action } func (sc *StepContext) execJobContainer() common.Executor { @@ -45,39 +48,37 @@ func (sc *StepContext) Executor() common.Executor { sc.runUsesContainer(), ) + case model.StepTypeUsesActionLocal: + return common.NewPipelineExecutor( + sc.setupEnv(), + sc.setupAction(), + sc.runAction(), + ) /* - case model.StepTypeUsesActionLocal: + case model.StepTypeUsesActionRemote: + remoteAction := newRemoteAction(step.Uses) + if remoteAction.Org == "actions" && remoteAction.Repo == "checkout" { + return func(ctx context.Context) error { + common.Logger(ctx).Debugf("Skipping actions/checkout") + return nil + } + } + cloneDir, err := ioutil.TempDir(rc.Tempdir, remoteAction.Repo) + if err != nil { + return common.NewErrorExecutor(err) + } return common.NewPipelineExecutor( + common.NewGitCloneExecutor(common.NewGitCloneExecutorInput{ + URL: remoteAction.CloneURL(), + Ref: remoteAction.Ref, + Dir: cloneDir, + }), sc.setupEnv(), sc.setupAction(), applyWith(containerSpec, step), rc.pullImage(containerSpec), rc.runContainer(containerSpec), ) - case model.StepTypeUsesActionRemote: - remoteAction := newRemoteAction(step.Uses) - if remoteAction.Org == "actions" && remoteAction.Repo == "checkout" { - return func(ctx context.Context) error { - common.Logger(ctx).Debugf("Skipping actions/checkout") - return nil - } - } - cloneDir, err := ioutil.TempDir(rc.Tempdir, remoteAction.Repo) - if err != nil { - return common.NewErrorExecutor(err) - } - return common.NewPipelineExecutor( - common.NewGitCloneExecutor(common.NewGitCloneExecutorInput{ - URL: remoteAction.CloneURL(), - Ref: remoteAction.Ref, - Dir: cloneDir, - }), - sc.setupEnv(), - sc.setupAction(), - applyWith(containerSpec, step), - rc.pullImage(containerSpec), - rc.runContainer(containerSpec), - ) */ } @@ -198,8 +199,6 @@ func (sc *StepContext) runUsesContainer() common.Executor { } } -/* - func (sc *StepContext) setupAction() common.Executor { rc := sc.RunContext step := sc.Step @@ -215,50 +214,94 @@ func (sc *StepContext) setupAction() common.Executor { return err } - action, err := model.ReadAction(f) - if err != nil { - return err - } + sc.Action, err = model.ReadAction(f) + log.Debugf("Read action %v from '%s'", sc.Action, f.Name()) + return err + } +} +func (sc *StepContext) runAction() common.Executor { + rc := sc.RunContext + step := sc.Step + return func(ctx context.Context) error { + action := sc.Action + log.Debugf("About to run action %v", action) for inputID, input := range action.Inputs { envKey := regexp.MustCompile("[^A-Z0-9-]").ReplaceAllString(strings.ToUpper(inputID), "_") envKey = fmt.Sprintf("INPUT_%s", envKey) - if _, ok := containerSpec.Env[envKey]; !ok { - containerSpec.Env[envKey] = input.Default + if _, ok := sc.Env[envKey]; !ok { + sc.Env[envKey] = input.Default } } switch action.Runs.Using { case model.ActionRunsUsingNode12: - if strings.HasPrefix(actionDir, rc.Config.Workdir) { - containerSpec.Entrypoint = fmt.Sprintf("node /github/workspace/%s/%s", strings.TrimPrefix(actionDir, rc.Config.Workdir), action.Runs.Main) - } else if strings.HasPrefix(actionDir, rc.Tempdir) { - containerSpec.Entrypoint = fmt.Sprintf("node /github/home/%s/%s", strings.TrimPrefix(actionDir, rc.Tempdir), action.Runs.Main) - } + return rc.execJobContainer([]string{"node", action.Runs.Main}, sc.Env)(ctx) case model.ActionRunsUsingDocker: - if strings.HasPrefix(actionDir, rc.Config.Workdir) { - containerSpec.Name = rc.createStepContainerName(strings.TrimPrefix(actionDir, rc.Config.Workdir)) - } else if strings.HasPrefix(actionDir, rc.Tempdir) { - containerSpec.Name = rc.createStepContainerName(strings.TrimPrefix(actionDir, rc.Tempdir)) - } - containerSpec.Reuse = rc.Config.ReuseContainers + var prepImage common.Executor + var image string if strings.HasPrefix(action.Runs.Image, "docker://") { - containerSpec.Image = strings.TrimPrefix(action.Runs.Image, "docker://") - containerSpec.Entrypoint = strings.Join(action.Runs.Entrypoint, " ") - containerSpec.Args = strings.Join(action.Runs.Args, " ") + image = strings.TrimPrefix(action.Runs.Image, "docker://") } else { - containerSpec.Image = fmt.Sprintf("%s:%s", containerSpec.Name, "latest") - contextDir := filepath.Join(actionDir, action.Runs.Main) - return container.NewDockerBuildExecutor(container.NewDockerBuildExecutorInput{ + image = fmt.Sprintf("%s:%s", regexp.MustCompile("[^a-zA-Z0-9]").ReplaceAllString(step.Uses, "-"), "latest") + image = strings.TrimLeft(image, "-") + contextDir := filepath.Join(rc.Config.Workdir, step.Uses, action.Runs.Main) + prepImage = container.NewDockerBuildExecutor(container.NewDockerBuildExecutorInput{ ContextDir: contextDir, - ImageTag: containerSpec.Image, - })(ctx) + ImageTag: image, + }) } + + cmd := strings.Fields(step.With["args"]) + if len(cmd) == 0 { + cmd = action.Runs.Args + } + entrypoint := strings.Fields(step.With["entrypoint"]) + if len(entrypoint) == 0 { + entrypoint = action.Runs.Entrypoint + } + stepContainer := sc.newStepContainer(ctx, image, cmd, entrypoint) + return common.NewPipelineExecutor( + prepImage, + stepContainer.Pull(rc.Config.ForcePull), + stepContainer.Remove().IfBool(!rc.Config.ReuseContainers), + stepContainer.Create(), + stepContainer.Start(true), + ).Finally( + stepContainer.Remove().IfBool(!rc.Config.ReuseContainers), + )(ctx) + + /* + case model.ActionRunsUsingNode12: + if strings.HasPrefix(actionDir, rc.Config.Workdir) { + containerSpec.Entrypoint = fmt.Sprintf("node /github/workspace/%s/%s", strings.TrimPrefix(actionDir, rc.Config.Workdir), action.Runs.Main) + } else if strings.HasPrefix(actionDir, rc.Tempdir) { + containerSpec.Entrypoint = fmt.Sprintf("node /github/home/%s/%s", strings.TrimPrefix(actionDir, rc.Tempdir), action.Runs.Main) + } + case model.ActionRunsUsingDocker: + if strings.HasPrefix(actionDir, rc.Config.Workdir) { + containerSpec.Name = rc.createStepContainerName(strings.TrimPrefix(actionDir, rc.Config.Workdir)) + } else if strings.HasPrefix(actionDir, rc.Tempdir) { + containerSpec.Name = rc.createStepContainerName(strings.TrimPrefix(actionDir, rc.Tempdir)) + } + containerSpec.Reuse = rc.Config.ReuseContainers + if strings.HasPrefix(action.Runs.Image, "docker://") { + containerSpec.Image = strings.TrimPrefix(action.Runs.Image, "docker://") + containerSpec.Entrypoint = strings.Join(action.Runs.Entrypoint, " ") + containerSpec.Args = strings.Join(action.Runs.Args, " ") + } else { + containerSpec.Image = fmt.Sprintf("%s:%s", containerSpec.Name, "latest") + contextDir := filepath.Join(actionDir, action.Runs.Main) + return container.NewDockerBuildExecutor(container.NewDockerBuildExecutorInput{ + ContextDir: contextDir, + ImageTag: containerSpec.Image, + })(ctx) + } + */ } return nil } } -*/ type remoteAction struct { Org string diff --git a/pkg/runner/testdata/basic/push.yml b/pkg/runner/testdata/basic/push.yml index 50c6553a..30218acd 100644 --- a/pkg/runner/testdata/basic/push.yml +++ b/pkg/runner/testdata/basic/push.yml @@ -10,7 +10,6 @@ jobs: - run: cat /github/sha.txt | grep ${GITHUB_SHA} build: - if: false runs-on: ubuntu-latest needs: [check] steps: From 88041afb87a9dce13624d4d03b6a0641611d7a38 Mon Sep 17 00:00:00 2001 From: Casey Lee Date: Sun, 23 Feb 2020 22:34:48 -0800 Subject: [PATCH 07/14] cache dir for remote actions --- pkg/container/docker_run.go | 2 +- pkg/container/docker_run_test.go | 52 ------------------ pkg/runner/expression.go | 11 ++++ pkg/runner/run_context.go | 31 +++++++---- pkg/runner/step_context.go | 91 +++++++++++++++----------------- 5 files changed, 76 insertions(+), 111 deletions(-) delete mode 100644 pkg/container/docker_run_test.go diff --git a/pkg/container/docker_run.go b/pkg/container/docker_run.go index 81a6464a..f964b411 100644 --- a/pkg/container/docker_run.go +++ b/pkg/container/docker_run.go @@ -171,7 +171,7 @@ func (cr *containerReference) remove() common.Executor { Force: true, }) if err != nil { - return errors.WithStack(err) + logger.Error(errors.WithStack(err)) } logger.Debugf("Removed container: %v", cr.id) diff --git a/pkg/container/docker_run_test.go b/pkg/container/docker_run_test.go deleted file mode 100644 index b68d329b..00000000 --- a/pkg/container/docker_run_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package container - -import ( - "bytes" - "context" - "io/ioutil" - "testing" - - "github.com/nektos/act/pkg/common" - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" -) - -type rawFormatter struct{} - -func (f *rawFormatter) Format(entry *logrus.Entry) ([]byte, error) { - return []byte(entry.Message), nil -} - -func TestNewDockerRunExecutor(t *testing.T) { - if testing.Short() { - t.Skip("skipping slower test") - } - - noopLogger := logrus.New() - noopLogger.SetOutput(ioutil.Discard) - - buf := &bytes.Buffer{} - logger := logrus.New() - logger.SetOutput(buf) - logger.SetFormatter(&rawFormatter{}) - - ctx := common.WithLogger(context.Background(), logger) - - runner := NewDockerRunExecutor(NewDockerRunExecutorInput{ - Image: "hello-world", - Stdout: buf, - }) - - puller := NewDockerPullExecutor(NewDockerPullExecutorInput{ - Image: "hello-world", - }) - - pipeline := common.NewPipelineExecutor(puller, runner) - err := pipeline(ctx) - assert.NoError(t, err) - - actual := buf.String() - assert.Contains(t, actual, `docker pull hello-world`) - assert.Contains(t, actual, `docker run image=hello-world entrypoint=[] cmd=[]`) - assert.Contains(t, actual, `Hello from Docker!`) -} diff --git a/pkg/runner/expression.go b/pkg/runner/expression.go index 9de525c5..652800ea 100644 --- a/pkg/runner/expression.go +++ b/pkg/runner/expression.go @@ -38,6 +38,7 @@ func (sc *StepContext) NewExpressionEvaluator() ExpressionEvaluator { vm := sc.RunContext.newVM() configers := []func(*otto.Otto){ sc.vmEnv(), + sc.vmInputs(), } for _, configer := range configers { configer(vm) @@ -241,6 +242,16 @@ func (sc *StepContext) vmEnv() func(*otto.Otto) { } } +func (sc *StepContext) vmInputs() func(*otto.Otto) { + inputs := make(map[string]string) + for k, v := range sc.Step.With { + inputs[k] = v + } + return func(vm *otto.Otto) { + _ = vm.Set("inputs", inputs) + } +} + func (rc *RunContext) vmJob() func(*otto.Otto) { job := rc.getJobContext() diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go index 81737250..10fe85cd 100644 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -45,7 +45,7 @@ func (rc *RunContext) GetEnv() map[string]string { } func (rc *RunContext) jobContainerName() string { - return createContainerName(filepath.Base(rc.Config.Workdir), rc.Run.String()) + return createContainerName("act", rc.Run.String()) } func (rc *RunContext) startJobContainer() common.Executor { @@ -74,7 +74,7 @@ func (rc *RunContext) startJobContainer() common.Executor { rc.JobContainer = container.NewContainer(&container.NewContainerInput{ Cmd: nil, - Entrypoint: []string{"/bin/cat"}, + Entrypoint: []string{"/usr/bin/tail", "-f", "/dev/null"}, WorkingDir: "/github/workspace", Image: image, Name: name, @@ -83,6 +83,7 @@ func (rc *RunContext) startJobContainer() common.Executor { }, Binds: []string{ fmt.Sprintf("%s:%s", rc.Config.Workdir, "/github/workspace"), + fmt.Sprintf("%s:%s", rc.ActionDir(), "/github/home/.act"), fmt.Sprintf("%s:%s", "/var/run/docker.sock", "/var/run/docker.sock"), }, Stdout: logWriter, @@ -98,10 +99,6 @@ func (rc *RunContext) startJobContainer() common.Executor { Name: "workflow/event.json", Mode: 644, Body: rc.EventJSON, - }, &container.FileEntry{ - Name: "home/.actions/.keep", - Mode: 644, - Body: "", }), )(ctx) } @@ -121,6 +118,18 @@ func (rc *RunContext) stopJobContainer() common.Executor { } } +// ActionDir is for rc +func (rc *RunContext) ActionDir() string { + var xdgCache string + var ok bool + if xdgCache, ok = os.LookupEnv("XDG_CACHE_HOME"); !ok { + if home, ok := os.LookupEnv("HOME"); ok { + xdgCache = fmt.Sprintf("%s/.cache", home) + } + } + return filepath.Join(xdgCache, "act") +} + // Executor returns a pipeline executor for all the steps in the job func (rc *RunContext) Executor() common.Executor { steps := make([]common.Executor, 0) @@ -210,10 +219,14 @@ func createContainerName(parts ...string) string { name := make([]string, 0) pattern := regexp.MustCompile("[^a-zA-Z0-9]") partLen := (30 / len(parts)) - 1 - for _, part := range parts { - name = append(name, trimToLen(pattern.ReplaceAllString(part, "-"), partLen)) + for i, part := range parts { + if i == len(parts)-1 { + name = append(name, pattern.ReplaceAllString(part, "-")) + } else { + name = append(name, trimToLen(pattern.ReplaceAllString(part, "-"), partLen)) + } } - return strings.Join(name, "-") + return trimToLen(strings.Trim(strings.Join(name, "-"), "-"), 30) } func trimToLen(s string, l int) string { diff --git a/pkg/runner/step_context.go b/pkg/runner/step_context.go index 2a252d00..9600afde 100644 --- a/pkg/runner/step_context.go +++ b/pkg/runner/step_context.go @@ -49,54 +49,37 @@ func (sc *StepContext) Executor() common.Executor { ) case model.StepTypeUsesActionLocal: + actionDir := filepath.Join(rc.Config.Workdir, step.Uses) return common.NewPipelineExecutor( sc.setupEnv(), - sc.setupAction(), - sc.runAction(), + sc.setupAction(actionDir), + sc.runAction(actionDir), + ) + case model.StepTypeUsesActionRemote: + remoteAction := newRemoteAction(step.Uses) + if remoteAction.Org == "actions" && remoteAction.Repo == "checkout" { + return func(ctx context.Context) error { + common.Logger(ctx).Debugf("Skipping actions/checkout") + return nil + } + } + + actionDir := rc.ActionDir() + return common.NewPipelineExecutor( + common.NewGitCloneExecutor(common.NewGitCloneExecutorInput{ + URL: remoteAction.CloneURL(), + Ref: remoteAction.Ref, + Dir: actionDir, + }), + sc.setupEnv(), + sc.setupAction(actionDir), + sc.runAction(actionDir), ) - /* - case model.StepTypeUsesActionRemote: - remoteAction := newRemoteAction(step.Uses) - if remoteAction.Org == "actions" && remoteAction.Repo == "checkout" { - return func(ctx context.Context) error { - common.Logger(ctx).Debugf("Skipping actions/checkout") - return nil - } - } - cloneDir, err := ioutil.TempDir(rc.Tempdir, remoteAction.Repo) - if err != nil { - return common.NewErrorExecutor(err) - } - return common.NewPipelineExecutor( - common.NewGitCloneExecutor(common.NewGitCloneExecutorInput{ - URL: remoteAction.CloneURL(), - Ref: remoteAction.Ref, - Dir: cloneDir, - }), - sc.setupEnv(), - sc.setupAction(), - applyWith(containerSpec, step), - rc.pullImage(containerSpec), - rc.runContainer(containerSpec), - ) - */ } return common.NewErrorExecutor(fmt.Errorf("Unable to determine how to run job:%s step:%+v", rc.Run, step)) } -func applyWith(containerSpec *model.ContainerSpec, step *model.Step) common.Executor { - return func(ctx context.Context) error { - if entrypoint, ok := step.With["entrypoint"]; ok { - containerSpec.Entrypoint = entrypoint - } - if args, ok := step.With["args"]; ok { - containerSpec.Args = args - } - return nil - } -} - func (sc *StepContext) setupEnv() common.Executor { rc := sc.RunContext job := rc.Run.Job() @@ -160,6 +143,13 @@ func (sc *StepContext) newStepContainer(ctx context.Context, image string, cmd [ for k, v := range sc.Env { envList = append(envList, fmt.Sprintf("%s=%s", k, v)) } + stepEE := sc.NewExpressionEvaluator() + for i, v := range cmd { + cmd[i] = stepEE.Interpolate(v) + } + for i, v := range entrypoint { + entrypoint[i] = stepEE.Interpolate(v) + } stepContainer := container.NewContainer(&container.NewContainerInput{ Cmd: cmd, Entrypoint: entrypoint, @@ -184,8 +174,8 @@ func (sc *StepContext) runUsesContainer() common.Executor { step := sc.Step return func(ctx context.Context) error { image := strings.TrimPrefix(step.Uses, "docker://") - cmd := strings.Fields(rc.ExprEval.Interpolate(step.With["args"])) - entrypoint := strings.Fields(rc.ExprEval.Interpolate(step.With["entrypoint"])) + cmd := strings.Fields(step.With["args"]) + entrypoint := strings.Fields(step.With["entrypoint"]) stepContainer := sc.newStepContainer(ctx, image, cmd, entrypoint) return common.NewPipelineExecutor( @@ -199,10 +189,7 @@ func (sc *StepContext) runUsesContainer() common.Executor { } } -func (sc *StepContext) setupAction() common.Executor { - rc := sc.RunContext - step := sc.Step - actionDir := filepath.Join(rc.Config.Workdir, step.Uses) +func (sc *StepContext) setupAction(actionDir string) common.Executor { return func(ctx context.Context) error { f, err := os.Open(filepath.Join(actionDir, "action.yml")) if os.IsNotExist(err) { @@ -220,7 +207,7 @@ func (sc *StepContext) setupAction() common.Executor { } } -func (sc *StepContext) runAction() common.Executor { +func (sc *StepContext) runAction(actionDir string) common.Executor { rc := sc.RunContext step := sc.Step return func(ctx context.Context) error { @@ -236,7 +223,13 @@ func (sc *StepContext) runAction() common.Executor { switch action.Runs.Using { case model.ActionRunsUsingNode12: - return rc.execJobContainer([]string{"node", action.Runs.Main}, sc.Env)(ctx) + basePath := "." + if strings.HasPrefix(actionDir, rc.Config.Workdir) { + basePath = fmt.Sprintf("/github/workspace/%s", strings.TrimPrefix(actionDir, rc.Config.Workdir)) + } else if strings.HasPrefix(actionDir, rc.ActionDir()) { + basePath = fmt.Sprintf("/github/home/.act/%s", strings.TrimPrefix(actionDir, rc.ActionDir())) + } + return rc.execJobContainer([]string{"node", fmt.Sprintf("%s/%s", basePath, action.Runs.Main)}, sc.Env)(ctx) case model.ActionRunsUsingDocker: var prepImage common.Executor var image string @@ -245,7 +238,7 @@ func (sc *StepContext) runAction() common.Executor { } else { image = fmt.Sprintf("%s:%s", regexp.MustCompile("[^a-zA-Z0-9]").ReplaceAllString(step.Uses, "-"), "latest") image = strings.TrimLeft(image, "-") - contextDir := filepath.Join(rc.Config.Workdir, step.Uses, action.Runs.Main) + contextDir := filepath.Join(actionDir, action.Runs.Main) prepImage = container.NewDockerBuildExecutor(container.NewDockerBuildExecutorInput{ ContextDir: contextDir, ImageTag: image, From 6c632946beff4facd3c50a7a09a718d063ddf369 Mon Sep 17 00:00:00 2001 From: Casey Lee Date: Mon, 24 Feb 2020 10:56:49 -0800 Subject: [PATCH 08/14] unit tests pass --- .github/workflows/push.yml | 2 +- pkg/runner/run_context.go | 30 +++++++++++++++++++++---- pkg/runner/runner_test.go | 6 ++++- pkg/runner/step_context.go | 45 ++++++++++++++++++++++++++++---------- 4 files changed, 66 insertions(+), 17 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index d1937db1..87d51e09 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -7,4 +7,4 @@ jobs: steps: - uses: actions/checkout@v2 - uses: ./.github/workflows/check - #- uses: ./.github/workflows/integration + - uses: ./.github/workflows/integration diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go index 10fe85cd..cd302cb9 100644 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "regexp" + "runtime" "strings" "github.com/nektos/act/pkg/container" @@ -72,18 +73,38 @@ func (rc *RunContext) startJobContainer() common.Executor { common.Logger(ctx).Infof("\U0001f680 Start image=%s", image) name := rc.jobContainerName() + envList := make([]string, 0) + bindModifiers := "" + if runtime.GOOS == "darwin" { + bindModifiers = ":delegated" + } + + hostWorkdir := os.Getenv("ACT_HOST_WORKDIR") + if hostWorkdir == "" { + hostWorkdir = rc.Config.Workdir + } + envList = append(envList, fmt.Sprintf("%s=%s", "ACT_HOST_WORKDIR", hostWorkdir)) + + hostActionCache := os.Getenv("ACT_HOST_ACTIONCACHE") + if hostActionCache == "" { + hostActionCache = rc.ActionCacheDir() + } + envList = append(envList, fmt.Sprintf("%s=%s", "ACT_HOST_ACTIONCACHE", hostActionCache)) + rc.JobContainer = container.NewContainer(&container.NewContainerInput{ Cmd: nil, Entrypoint: []string{"/usr/bin/tail", "-f", "/dev/null"}, WorkingDir: "/github/workspace", Image: image, Name: name, + Env: envList, Mounts: map[string]string{ name: "/github", }, + Binds: []string{ - fmt.Sprintf("%s:%s", rc.Config.Workdir, "/github/workspace"), - fmt.Sprintf("%s:%s", rc.ActionDir(), "/github/home/.act"), + fmt.Sprintf("%s:%s%s", hostWorkdir, "/github/workspace", bindModifiers), + fmt.Sprintf("%s:%s%s", hostActionCache, "/github/home/.cache/act", bindModifiers), fmt.Sprintf("%s:%s", "/var/run/docker.sock", "/var/run/docker.sock"), }, Stdout: logWriter, @@ -118,8 +139,8 @@ func (rc *RunContext) stopJobContainer() common.Executor { } } -// ActionDir is for rc -func (rc *RunContext) ActionDir() string { +// ActionCacheDir is for rc +func (rc *RunContext) ActionCacheDir() string { var xdgCache string var ok bool if xdgCache, ok = os.LookupEnv("XDG_CACHE_HOME"); !ok { @@ -336,6 +357,7 @@ func (rc *RunContext) withGithubEnv(env map[string]string) map[string]string { env["GITHUB_RUN_ID"] = github.RunID env["GITHUB_RUN_NUMBER"] = github.RunNumber env["GITHUB_ACTION"] = github.Action + env["GITHUB_ACTIONS"] = "true" env["GITHUB_ACTOR"] = github.Actor env["GITHUB_REPOSITORY"] = github.Repository env["GITHUB_EVENT_NAME"] = github.EventName diff --git a/pkg/runner/runner_test.go b/pkg/runner/runner_test.go index e4fb880f..3be32aec 100644 --- a/pkg/runner/runner_test.go +++ b/pkg/runner/runner_test.go @@ -3,6 +3,7 @@ package runner import ( "context" "fmt" + "path/filepath" "testing" "github.com/nektos/act/pkg/model" @@ -59,8 +60,11 @@ func TestRunEvent(t *testing.T) { platforms := map[string]string{ "ubuntu-latest": "node:12.6-buster-slim", } + + workdir, err := filepath.Abs("testdata") + assert.NilError(t, err, table.workflowPath) runnerConfig := &Config{ - Workdir: "testdata", + Workdir: workdir, EventName: table.eventName, Platforms: platforms, ReuseContainers: false, diff --git a/pkg/runner/step_context.go b/pkg/runner/step_context.go index 9600afde..479998f4 100644 --- a/pkg/runner/step_context.go +++ b/pkg/runner/step_context.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "regexp" + "runtime" "strings" "github.com/nektos/act/pkg/common" @@ -64,7 +65,7 @@ func (sc *StepContext) Executor() common.Executor { } } - actionDir := rc.ActionDir() + actionDir := fmt.Sprintf("%s/%s", rc.ActionCacheDir(), remoteAction.Repo) return common.NewPipelineExecutor( common.NewGitCloneExecutor(common.NewGitCloneExecutorInput{ URL: remoteAction.CloneURL(), @@ -150,6 +151,24 @@ func (sc *StepContext) newStepContainer(ctx context.Context, image string, cmd [ for i, v := range entrypoint { entrypoint[i] = stepEE.Interpolate(v) } + + bindModifiers := "" + if runtime.GOOS == "darwin" { + bindModifiers = ":delegated" + } + + hostWorkdir := os.Getenv("ACT_HOST_WORKDIR") + if hostWorkdir == "" { + hostWorkdir = rc.Config.Workdir + } + envList = append(envList, fmt.Sprintf("%s=%s", "ACT_HOST_WORKDIR", hostWorkdir)) + + hostActionCache := os.Getenv("ACT_HOST_ACTIONCACHE") + if hostActionCache == "" { + hostActionCache = rc.ActionCacheDir() + } + envList = append(envList, fmt.Sprintf("%s=%s", "ACT_HOST_ACTIONCACHE", hostActionCache)) + stepContainer := container.NewContainer(&container.NewContainerInput{ Cmd: cmd, Entrypoint: entrypoint, @@ -161,7 +180,7 @@ func (sc *StepContext) newStepContainer(ctx context.Context, image string, cmd [ rc.jobContainerName(): "/github", }, Binds: []string{ - fmt.Sprintf("%s:%s", rc.Config.Workdir, "/github/workspace"), + fmt.Sprintf("%s:%s%s", hostWorkdir, "/github/workspace", bindModifiers), fmt.Sprintf("%s:%s", "/var/run/docker.sock", "/var/run/docker.sock"), }, Stdout: logWriter, @@ -221,23 +240,27 @@ func (sc *StepContext) runAction(actionDir string) common.Executor { } } + actionName := "" + containerActionDir := "." + if strings.HasPrefix(actionDir, rc.Config.Workdir) { + actionName = strings.TrimPrefix(actionDir, rc.Config.Workdir) + containerActionDir = "/github/workspace" + } else if strings.HasPrefix(actionDir, rc.ActionCacheDir()) { + actionName = strings.TrimPrefix(actionDir, rc.ActionCacheDir()) + containerActionDir = "/github/home/.cache/act" + } + switch action.Runs.Using { case model.ActionRunsUsingNode12: - basePath := "." - if strings.HasPrefix(actionDir, rc.Config.Workdir) { - basePath = fmt.Sprintf("/github/workspace/%s", strings.TrimPrefix(actionDir, rc.Config.Workdir)) - } else if strings.HasPrefix(actionDir, rc.ActionDir()) { - basePath = fmt.Sprintf("/github/home/.act/%s", strings.TrimPrefix(actionDir, rc.ActionDir())) - } - return rc.execJobContainer([]string{"node", fmt.Sprintf("%s/%s", basePath, action.Runs.Main)}, sc.Env)(ctx) + return rc.execJobContainer([]string{"node", fmt.Sprintf("%s/%s/%s", containerActionDir, actionName, action.Runs.Main)}, sc.Env)(ctx) case model.ActionRunsUsingDocker: var prepImage common.Executor var image string if strings.HasPrefix(action.Runs.Image, "docker://") { image = strings.TrimPrefix(action.Runs.Image, "docker://") } else { - image = fmt.Sprintf("%s:%s", regexp.MustCompile("[^a-zA-Z0-9]").ReplaceAllString(step.Uses, "-"), "latest") - image = strings.TrimLeft(image, "-") + image = fmt.Sprintf("%s:%s", regexp.MustCompile("[^a-zA-Z0-9]").ReplaceAllString(actionName, "-"), "latest") + image = fmt.Sprintf("act-%s", strings.TrimLeft(image, "-")) contextDir := filepath.Join(actionDir, action.Runs.Main) prepImage = container.NewDockerBuildExecutor(container.NewDockerBuildExecutorInput{ ContextDir: contextDir, From 037e08a3a77eb3d89d765213617e97e9691d83e3 Mon Sep 17 00:00:00 2001 From: Casey Lee Date: Mon, 24 Feb 2020 12:48:12 -0800 Subject: [PATCH 09/14] integration test --- .github/workflows/{check => lint}/Dockerfile | 0 .github/workflows/{check => lint}/action.yml | 4 +- .../workflows/{check => lint}/entrypoint.sh | 1 - .github/workflows/push.yml | 9 +- .github/workflows/tag.yml | 2 - pkg/common/git.go | 2 +- pkg/common/line_writer.go | 7 +- pkg/common/line_writer_test.go | 3 +- pkg/runner/command.go | 89 +++++++++++-------- pkg/runner/command_test.go | 13 +++ pkg/runner/run_context.go | 19 ++-- pkg/runner/step_context.go | 18 ++-- 12 files changed, 94 insertions(+), 73 deletions(-) rename .github/workflows/{check => lint}/Dockerfile (100%) rename .github/workflows/{check => lint}/action.yml (60%) rename .github/workflows/{check => lint}/entrypoint.sh (56%) diff --git a/.github/workflows/check/Dockerfile b/.github/workflows/lint/Dockerfile similarity index 100% rename from .github/workflows/check/Dockerfile rename to .github/workflows/lint/Dockerfile diff --git a/.github/workflows/check/action.yml b/.github/workflows/lint/action.yml similarity index 60% rename from .github/workflows/check/action.yml rename to .github/workflows/lint/action.yml index 5222118d..2f300875 100644 --- a/.github/workflows/check/action.yml +++ b/.github/workflows/lint/action.yml @@ -1,5 +1,5 @@ -name: Check -description: Run static analysis and unit tests +name: Lint +description: Run static analysis branding: icon: check-circle color: green diff --git a/.github/workflows/check/entrypoint.sh b/.github/workflows/lint/entrypoint.sh similarity index 56% rename from .github/workflows/check/entrypoint.sh rename to .github/workflows/lint/entrypoint.sh index bbfc168d..304b1b98 100644 --- a/.github/workflows/check/entrypoint.sh +++ b/.github/workflows/lint/entrypoint.sh @@ -1,4 +1,3 @@ #!/bin/sh set -e golangci-lint run -go test -cover -short ./... diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 87d51e09..0db51379 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -6,5 +6,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: ./.github/workflows/check - - uses: ./.github/workflows/integration + - uses: ./.github/workflows/lint + - uses: actions/setup-go@v1 + with: + go-version: 1.13 + - run: go test -cover ./... + env: + CGO_ENABLED: 0 diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml index f64b244a..6c1f4ef4 100644 --- a/.github/workflows/tag.yml +++ b/.github/workflows/tag.yml @@ -9,8 +9,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: ./.github/workflows/check - #- uses: ./.github/workflows/integration - name: Run GoReleaser uses: goreleaser/goreleaser-action@v1 with: diff --git a/pkg/common/git.go b/pkg/common/git.go index 159141fa..c7a059ca 100644 --- a/pkg/common/git.go +++ b/pkg/common/git.go @@ -254,7 +254,7 @@ func NewGitCloneExecutor(input NewGitCloneExecutorInput) Executor { Force: true, }) if err != nil { - logger.Errorf("Unable to checkout %s: %v", refName, err) + logger.Errorf("Unable to checkout %s: %v", *hash, err) return err } diff --git a/pkg/common/line_writer.go b/pkg/common/line_writer.go index 4d1661b2..20351997 100644 --- a/pkg/common/line_writer.go +++ b/pkg/common/line_writer.go @@ -6,7 +6,7 @@ import ( ) // LineHandler is a callback function for handling a line -type LineHandler func(line string) +type LineHandler func(line string) bool type lineWriter struct { buffer bytes.Buffer @@ -42,6 +42,9 @@ func (lw *lineWriter) Write(p []byte) (n int, err error) { func (lw *lineWriter) handleLine(line string) { for _, h := range lw.handlers { - h(line) + ok := h(line) + if !ok { + break + } } } diff --git a/pkg/common/line_writer_test.go b/pkg/common/line_writer_test.go index 462a20d9..44e11ef5 100644 --- a/pkg/common/line_writer_test.go +++ b/pkg/common/line_writer_test.go @@ -8,8 +8,9 @@ import ( func TestLineWriter(t *testing.T) { lines := make([]string, 0) - lineHandler := func(s string) { + lineHandler := func(s string) bool { lines = append(lines, s) + return true } lineWriter := NewLineWriter(lineHandler) diff --git a/pkg/runner/command.go b/pkg/runner/command.go index 08c1ac90..9e27b742 100644 --- a/pkg/runner/command.go +++ b/pkg/runner/command.go @@ -8,50 +8,63 @@ import ( "github.com/nektos/act/pkg/common" ) -var commandPattern *regexp.Regexp +var commandPatternGA *regexp.Regexp +var commandPatternADO *regexp.Regexp func init() { - commandPattern = regexp.MustCompile("^::([^ ]+)( (.+))?::([^\r\n]*)[\r\n]+$") + commandPatternGA = regexp.MustCompile("^::([^ ]+)( (.+))?::([^\r\n]*)[\r\n]+$") + commandPatternADO = regexp.MustCompile("^##\\[([^ ]+)( (.+))?\\]([^\r\n]*)[\r\n]+$") } func (rc *RunContext) commandHandler(ctx context.Context) common.LineHandler { logger := common.Logger(ctx) resumeCommand := "" - return func(line string) { - if m := commandPattern.FindStringSubmatch(line); m != nil { - command := m[1] - kvPairs := parseKeyValuePairs(m[3]) - arg := m[4] - - if resumeCommand != "" && command != resumeCommand { - return - } - - switch command { - case "set-env": - rc.setEnv(ctx, kvPairs, arg) - case "set-output": - rc.setOutput(ctx, kvPairs, arg) - case "add-path": - rc.addPath(ctx, arg) - case "debug": - logger.Infof(" \U0001F4AC %s", line) - case "warning": - logger.Infof(" \U0001F6A7 %s", line) - case "error": - logger.Infof(" \U00002757 %s", line) - case "add-mask": - logger.Infof(" \U00002699 %s", line) - case "stop-commands": - resumeCommand = arg - logger.Infof(" \U00002699 %s", line) - case resumeCommand: - resumeCommand = "" - logger.Infof(" \U00002699 %s", line) - default: - logger.Infof(" \U00002753 %s", line) - } + return func(line string) bool { + var command string + var kvPairs map[string]string + var arg string + if m := commandPatternGA.FindStringSubmatch(line); m != nil { + command = m[1] + kvPairs = parseKeyValuePairs(m[3], ",") + arg = m[4] + } else if m := commandPatternADO.FindStringSubmatch(line); m != nil { + command = m[1] + kvPairs = parseKeyValuePairs(m[3], ";") + arg = m[4] + } else { + return true } + + if resumeCommand != "" && command != resumeCommand { + return false + } + + switch command { + case "set-env": + rc.setEnv(ctx, kvPairs, arg) + case "set-output": + rc.setOutput(ctx, kvPairs, arg) + case "add-path": + rc.addPath(ctx, arg) + case "debug": + logger.Infof(" \U0001F4AC %s", line) + case "warning": + logger.Infof(" \U0001F6A7 %s", line) + case "error": + logger.Infof(" \U00002757 %s", line) + case "add-mask": + logger.Infof(" \U00002699 %s", line) + case "stop-commands": + resumeCommand = arg + logger.Infof(" \U00002699 %s", line) + case resumeCommand: + resumeCommand = "" + logger.Infof(" \U00002699 %s", line) + default: + logger.Infof(" \U00002753 %s", line) + } + + return false } } @@ -71,9 +84,9 @@ func (rc *RunContext) addPath(ctx context.Context, arg string) { rc.ExtraPath = append(rc.ExtraPath, arg) } -func parseKeyValuePairs(kvPairs string) map[string]string { +func parseKeyValuePairs(kvPairs string, separator string) map[string]string { rtn := make(map[string]string) - kvPairList := strings.Split(kvPairs, ",") + kvPairList := strings.Split(kvPairs, separator) for _, kvPair := range kvPairList { kv := strings.Split(kvPair, "=") if len(kv) == 2 { diff --git a/pkg/runner/command_test.go b/pkg/runner/command_test.go index 4279bc02..6abe5275 100644 --- a/pkg/runner/command_test.go +++ b/pkg/runner/command_test.go @@ -60,3 +60,16 @@ func TestStopCommands(t *testing.T) { handler("::set-env name=x::abcd\n") assert.Equal("abcd", rc.Env["x"]) } + +func TestAddpathADO(t *testing.T) { + assert := assert.New(t) + ctx := context.Background() + rc := new(RunContext) + handler := rc.commandHandler(ctx) + + handler("##[add-path]/zoo\n") + assert.Equal("/zoo", rc.ExtraPath[0]) + + handler("##[add-path]/boo\n") + assert.Equal("/boo", rc.ExtraPath[1]) +} diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go index cd302cb9..a86a0090 100644 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -62,12 +62,13 @@ func (rc *RunContext) startJobContainer() common.Executor { return func(ctx context.Context) error { rawLogger := common.Logger(ctx).WithField("raw_output", true) - logWriter := common.NewLineWriter(rc.commandHandler(ctx), func(s string) { + logWriter := common.NewLineWriter(rc.commandHandler(ctx), func(s string) bool { if rc.Config.LogOutput { rawLogger.Infof(s) } else { rawLogger.Debugf(s) } + return true }) common.Logger(ctx).Infof("\U0001f680 Start image=%s", image) @@ -79,17 +80,12 @@ func (rc *RunContext) startJobContainer() common.Executor { bindModifiers = ":delegated" } - hostWorkdir := os.Getenv("ACT_HOST_WORKDIR") - if hostWorkdir == "" { - hostWorkdir = rc.Config.Workdir - } - envList = append(envList, fmt.Sprintf("%s=%s", "ACT_HOST_WORKDIR", hostWorkdir)) - - hostActionCache := os.Getenv("ACT_HOST_ACTIONCACHE") + hostActionCache := os.Getenv("ACT_HOST_ACTION_CACHE") if hostActionCache == "" { hostActionCache = rc.ActionCacheDir() } - envList = append(envList, fmt.Sprintf("%s=%s", "ACT_HOST_ACTIONCACHE", hostActionCache)) + envList = append(envList, fmt.Sprintf("%s=%s", "ACT_HOST_ACTION_CACHE", hostActionCache)) + envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_TOOL_CACHE", "/toolcache")) rc.JobContainer = container.NewContainer(&container.NewContainerInput{ Cmd: nil, @@ -99,11 +95,12 @@ func (rc *RunContext) startJobContainer() common.Executor { Name: name, Env: envList, Mounts: map[string]string{ - name: "/github", + name: "/github", + "act-toolcache": "/toolcache", }, Binds: []string{ - fmt.Sprintf("%s:%s%s", hostWorkdir, "/github/workspace", bindModifiers), + fmt.Sprintf("%s:%s%s", rc.Config.Workdir, "/github/workspace", bindModifiers), fmt.Sprintf("%s:%s%s", hostActionCache, "/github/home/.cache/act", bindModifiers), fmt.Sprintf("%s:%s", "/var/run/docker.sock", "/var/run/docker.sock"), }, diff --git a/pkg/runner/step_context.go b/pkg/runner/step_context.go index 479998f4..a78211cf 100644 --- a/pkg/runner/step_context.go +++ b/pkg/runner/step_context.go @@ -133,12 +133,13 @@ func (sc *StepContext) newStepContainer(ctx context.Context, image string, cmd [ rc := sc.RunContext step := sc.Step rawLogger := common.Logger(ctx).WithField("raw_output", true) - logWriter := common.NewLineWriter(rc.commandHandler(ctx), func(s string) { + logWriter := common.NewLineWriter(rc.commandHandler(ctx), func(s string) bool { if rc.Config.LogOutput { rawLogger.Infof(s) } else { rawLogger.Debugf(s) } + return true }) envList := make([]string, 0) for k, v := range sc.Env { @@ -157,17 +158,7 @@ func (sc *StepContext) newStepContainer(ctx context.Context, image string, cmd [ bindModifiers = ":delegated" } - hostWorkdir := os.Getenv("ACT_HOST_WORKDIR") - if hostWorkdir == "" { - hostWorkdir = rc.Config.Workdir - } - envList = append(envList, fmt.Sprintf("%s=%s", "ACT_HOST_WORKDIR", hostWorkdir)) - - hostActionCache := os.Getenv("ACT_HOST_ACTIONCACHE") - if hostActionCache == "" { - hostActionCache = rc.ActionCacheDir() - } - envList = append(envList, fmt.Sprintf("%s=%s", "ACT_HOST_ACTIONCACHE", hostActionCache)) + envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_TOOL_CACHE", "/toolcache")) stepContainer := container.NewContainer(&container.NewContainerInput{ Cmd: cmd, @@ -178,9 +169,10 @@ func (sc *StepContext) newStepContainer(ctx context.Context, image string, cmd [ Env: envList, Mounts: map[string]string{ rc.jobContainerName(): "/github", + "act-toolcache": "/toolcache", }, Binds: []string{ - fmt.Sprintf("%s:%s%s", hostWorkdir, "/github/workspace", bindModifiers), + fmt.Sprintf("%s:%s%s", rc.Config.Workdir, "/github/workspace", bindModifiers), fmt.Sprintf("%s:%s", "/var/run/docker.sock", "/var/run/docker.sock"), }, Stdout: logWriter, From 3a65967b95a0de8188e00fc0f28253b6e0bdca67 Mon Sep 17 00:00:00 2001 From: Casey Lee Date: Mon, 24 Feb 2020 14:43:03 -0800 Subject: [PATCH 10/14] ci cleanup --- .github/workflows/integration/Dockerfile | 7 ------- .github/workflows/integration/action.yml | 8 -------- .github/workflows/integration/entrypoint.sh | 3 --- .github/workflows/lint/Dockerfile | 9 --------- .github/workflows/lint/action.yml | 8 -------- .github/workflows/lint/entrypoint.sh | 3 --- .github/workflows/push.yml | 15 +++++++++++++-- .github/workflows/tag.yml | 3 ++- Makefile | 5 ++--- 9 files changed, 17 insertions(+), 44 deletions(-) delete mode 100644 .github/workflows/integration/Dockerfile delete mode 100644 .github/workflows/integration/action.yml delete mode 100644 .github/workflows/integration/entrypoint.sh delete mode 100644 .github/workflows/lint/Dockerfile delete mode 100644 .github/workflows/lint/action.yml delete mode 100644 .github/workflows/lint/entrypoint.sh diff --git a/.github/workflows/integration/Dockerfile b/.github/workflows/integration/Dockerfile deleted file mode 100644 index 9a5c1083..00000000 --- a/.github/workflows/integration/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM golangci/golangci-lint:v1.23.6 - -COPY "entrypoint.sh" "/entrypoint.sh" -RUN chmod +x /entrypoint.sh - -ENV GOFLAGS -mod=vendor -ENTRYPOINT ["/entrypoint.sh"] diff --git a/.github/workflows/integration/action.yml b/.github/workflows/integration/action.yml deleted file mode 100644 index 8c73f49a..00000000 --- a/.github/workflows/integration/action.yml +++ /dev/null @@ -1,8 +0,0 @@ -name: Check -description: Run integration tests -branding: - icon: check-circle - color: green -runs: - using: 'docker' - image: 'Dockerfile' diff --git a/.github/workflows/integration/entrypoint.sh b/.github/workflows/integration/entrypoint.sh deleted file mode 100644 index 262d0515..00000000 --- a/.github/workflows/integration/entrypoint.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -set -e -go test -cover ./pkg/runner diff --git a/.github/workflows/lint/Dockerfile b/.github/workflows/lint/Dockerfile deleted file mode 100644 index 5c67ab28..00000000 --- a/.github/workflows/lint/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM golangci/golangci-lint:v1.23.6 - -RUN apt-get install git - -COPY "entrypoint.sh" "/entrypoint.sh" -RUN chmod +x /entrypoint.sh - -ENV GOFLAGS -mod=vendor -ENTRYPOINT ["/entrypoint.sh"] diff --git a/.github/workflows/lint/action.yml b/.github/workflows/lint/action.yml deleted file mode 100644 index 2f300875..00000000 --- a/.github/workflows/lint/action.yml +++ /dev/null @@ -1,8 +0,0 @@ -name: Lint -description: Run static analysis -branding: - icon: check-circle - color: green -runs: - using: 'docker' - image: 'Dockerfile' diff --git a/.github/workflows/lint/entrypoint.sh b/.github/workflows/lint/entrypoint.sh deleted file mode 100644 index 304b1b98..00000000 --- a/.github/workflows/lint/entrypoint.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -set -e -golangci-lint run diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 0db51379..66c241a2 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -2,14 +2,25 @@ name: push on: push jobs: - ci: + lint: + runs-on: ubuntu-latest + steps: + - run: echo ${{github.ref}} + - uses: actions/checkout@v2 + - uses: docker://golangci/golangci-lint:v1.23.6 + with: + args: golangci-lint run + env: + CGO_ENABLED: 0 + + test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: ./.github/workflows/lint - uses: actions/setup-go@v1 with: go-version: 1.13 - run: go test -cover ./... env: CGO_ENABLED: 0 + GOFLAGS: -mod=vendor diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml index 6c1f4ef4..f258c22b 100644 --- a/.github/workflows/tag.yml +++ b/.github/workflows/tag.yml @@ -6,10 +6,11 @@ on: jobs: release: + if: startsWith(github.ref, "v") runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Run GoReleaser + - name: GoReleaser uses: goreleaser/goreleaser-action@v1 with: version: latest diff --git a/Makefile b/Makefile index c8fd328c..d3e23877 100644 --- a/Makefile +++ b/Makefile @@ -12,12 +12,11 @@ endif IS_SNAPSHOT = $(if $(findstring -, $(VERSION)),true,false) TAG_VERSION = v$(VERSION) -ACT ?= go run -mod=vendor main.go +ACT ?= go run main.go export GITHUB_TOKEN = $(shell cat ~/.config/github/token) check: - @golangci-lint run - @go test -cover ./... + $(ACT) -P ubuntu-latest=nektos/act-environments-ubuntu:18.04 build: check $(eval export SNAPSHOT_VERSION=$(VERSION)) From d54b25d983155889fde45f0d08fc6dfb4e631aeb Mon Sep 17 00:00:00 2001 From: Casey Lee Date: Mon, 24 Feb 2020 15:04:33 -0800 Subject: [PATCH 11/14] makefile cleanups --- .github/workflows/push.yml | 17 ++++++++++++++++- .github/workflows/tag.yml | 19 ------------------- Makefile | 36 ++++++++++++++---------------------- 3 files changed, 30 insertions(+), 42 deletions(-) delete mode 100644 .github/workflows/tag.yml diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 66c241a2..7abd6010 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -5,7 +5,6 @@ jobs: lint: runs-on: ubuntu-latest steps: - - run: echo ${{github.ref}} - uses: actions/checkout@v2 - uses: docker://golangci/golangci-lint:v1.23.6 with: @@ -24,3 +23,19 @@ jobs: env: CGO_ENABLED: 0 GOFLAGS: -mod=vendor + + release: + if: startsWith(github.ref, "refs/tags/v") + needs: + - lint + - test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: GoReleaser + uses: goreleaser/goreleaser-action@v1 + with: + version: latest + args: release --rm-dist + env: + GITHUB_TOKEN: ${{ secrets.GORELEASER_GITHUB_TOKEN }} diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml deleted file mode 100644 index f258c22b..00000000 --- a/.github/workflows/tag.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: tag -on: - push: - tags: - - 'v*' - -jobs: - release: - if: startsWith(github.ref, "v") - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: GoReleaser - uses: goreleaser/goreleaser-action@v1 - with: - version: latest - args: release --rm-dist - env: - GITHUB_TOKEN: ${{ secrets.GORELEASER_GITHUB_TOKEN }} diff --git a/Makefile b/Makefile index d3e23877..abd9ec83 100644 --- a/Makefile +++ b/Makefile @@ -1,29 +1,21 @@ -LATEST_VERSION := $(shell git tag -l --sort=creatordate | grep "^v[0-9]*.[0-9]*.[0-9]*$$" | tail -1 | cut -c 2-) -ifeq "$(shell git tag -l v$(LATEST_VERSION) --points-at HEAD)" "v$(LATEST_VERSION)" -### latest tag points to current commit, this is a release build -VERSION ?= $(LATEST_VERSION) -else -### latest tag points to prior commit, this is a snapshot build -MAJOR_VERSION := $(word 1, $(subst ., ,$(LATEST_VERSION))) -MINOR_VERSION := $(word 2, $(subst ., ,$(LATEST_VERSION))) -PATCH_VERSION := $(word 3, $(subst ., ,$(LATEST_VERSION))) -VERSION ?= $(MAJOR_VERSION).$(MINOR_VERSION).$(shell echo $$(( $(PATCH_VERSION) + 1)) )-develop -endif +VERSION?=$(shell git describe --tags --dirty | cut -c 2-) IS_SNAPSHOT = $(if $(findstring -, $(VERSION)),true,false) -TAG_VERSION = v$(VERSION) +MAJOR_VERSION = $(word 1, $(subst ., ,$(VERSION))) +MINOR_VERSION = $(word 2, $(subst ., ,$(VERSION))) +PATCH_VERSION = $(word 3, $(subst ., ,$(word 1,$(subst -, , $(VERSION))))) +NEW_VERSION ?= $(MAJOR_VERSION).$(MINOR_VERSION).$(shell echo $$(( $(PATCH_VERSION) + 1)) ) ACT ?= go run main.go export GITHUB_TOKEN = $(shell cat ~/.config/github/token) -check: +build: + go build -ldflags "-X main.version=$(VERSION)" -o dist/local/act main.go + +test: $(ACT) -P ubuntu-latest=nektos/act-environments-ubuntu:18.04 -build: check - $(eval export SNAPSHOT_VERSION=$(VERSION)) - $(ACT) -ra build - install: build - @cp dist/$(shell go env GOOS)_$(shell go env GOARCH)/act /usr/local/bin/act + @cp dist/local/act /usr/local/bin/act @chmod 755 /usr/local/bin/act @act --version @@ -32,7 +24,8 @@ installer: godownloader -r nektos/act -o install.sh promote: vendor - @echo "VERSION:$(VERSION) IS_SNAPSHOT:$(IS_SNAPSHOT) LATEST_VERSION:$(LATEST_VERSION)" + @git fetch --tags + @echo "VERSION:$(VERSION) IS_SNAPSHOT:$(IS_SNAPSHOT) NEW_VERSION:$(NEW_VERSION)" ifeq (false,$(IS_SNAPSHOT)) @echo "Unable to promote a non-snapshot" @exit 1 @@ -41,9 +34,8 @@ ifneq ($(shell git status -s),) @echo "Unable to promote a dirty workspace" @exit 1 endif - $(eval NEW_VERSION := $(word 1,$(subst -, , $(TAG_VERSION)))) - git tag -a -m "releasing $(NEW_VERSION)" $(NEW_VERSION) - git push origin $(NEW_VERSION) + git tag -a -m "releasing v$(NEW_VERSION)" v$(NEW_VERSION) + git push origin v$(NEW_VERSION) vendor: go mod vendor From eb28924ba5a9b6725f6a3736912c5ff4d51466c5 Mon Sep 17 00:00:00 2001 From: Casey Lee Date: Mon, 24 Feb 2020 15:05:41 -0800 Subject: [PATCH 12/14] actions --- .github/workflows/push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 7abd6010..78450ff0 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -25,7 +25,7 @@ jobs: GOFLAGS: -mod=vendor release: - if: startsWith(github.ref, "refs/tags/v") + if: startsWith(github.ref, 'refs/tags/v') needs: - lint - test From 8f5918942d58af076d6203bb136708f74618ceb1 Mon Sep 17 00:00:00 2001 From: Casey Lee Date: Mon, 24 Feb 2020 16:38:49 -0800 Subject: [PATCH 13/14] move action cache to volume --- pkg/container/docker_run.go | 90 +++++++++++++++++++++++++++++++++++++ pkg/runner/run_context.go | 11 +++-- pkg/runner/step_context.go | 49 +++++++------------- 3 files changed, 110 insertions(+), 40 deletions(-) diff --git a/pkg/container/docker_run.go b/pkg/container/docker_run.go index f964b411..f7ee069a 100644 --- a/pkg/container/docker_run.go +++ b/pkg/container/docker_run.go @@ -6,7 +6,10 @@ import ( "context" "fmt" "io" + "io/ioutil" "os" + "path/filepath" + "strings" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" @@ -44,6 +47,7 @@ type FileEntry struct { type Container interface { Create() common.Executor Copy(destPath string, files ...*FileEntry) common.Executor + CopyDir(destPath string, srcPath string) common.Executor Pull(forcePull bool) common.Executor Start(attach bool) common.Executor Exec(command []string, env map[string]string) common.Executor @@ -95,6 +99,14 @@ func (cr *containerReference) Copy(destPath string, files ...*FileEntry) common. ).IfNot(common.Dryrun) } +func (cr *containerReference) CopyDir(destPath string, srcPath string) common.Executor { + return common.NewPipelineExecutor( + cr.connect(), + cr.find(), + cr.copyDir(destPath, srcPath), + ).IfNot(common.Dryrun) +} + func (cr *containerReference) Exec(command []string, env map[string]string) common.Executor { return common.NewPipelineExecutor( @@ -289,6 +301,84 @@ func (cr *containerReference) exec(cmd []string, env map[string]string) common.E } } +func (cr *containerReference) copyDir(dstPath string, srcPath string) common.Executor { + return func(ctx context.Context) error { + logger := common.Logger(ctx) + tarFile, err := ioutil.TempFile("", "act") + if err != nil { + return err + } + log.Debugf("Writing tarball %s from %s", tarFile.Name(), srcPath) + defer tarFile.Close() + //defer os.Remove(tarFile.Name()) + tw := tar.NewWriter(tarFile) + + srcPrefix := filepath.Dir(srcPath) + if !strings.HasSuffix(srcPrefix, string(filepath.Separator)) { + srcPrefix += string(filepath.Separator) + } + + err = filepath.Walk(srcPath, func(file string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + + // return on non-regular files (thanks to [kumo](https://medium.com/@komuw/just-like-you-did-fbdd7df829d3) for this suggested update) + if !fi.Mode().IsRegular() { + return nil + } + + // create a new dir/file header + header, err := tar.FileInfoHeader(fi, fi.Name()) + if err != nil { + return err + } + + // update the name to correctly reflect the desired destination when untaring + header.Name = strings.TrimPrefix(file, srcPrefix) + + // write the header + if err := tw.WriteHeader(header); err != nil { + return err + } + + // open files for taring + f, err := os.Open(file) + if err != nil { + return err + } + + // copy file data into tar writer + if _, err := io.Copy(tw, f); err != nil { + return err + } + + // manually close here after each file operation; defering would cause each file close + // to wait until all operations have completed. + f.Close() + + return nil + }) + if err != nil { + return err + } + if err := tw.Close(); err != nil { + return err + } + + logger.Debugf("Extracting content from '%s' to '%s'", tarFile.Name(), dstPath) + _, err = tarFile.Seek(0, 0) + if err != nil { + return errors.WithStack(err) + } + err = cr.cli.CopyToContainer(ctx, cr.id, dstPath, tarFile, types.CopyToContainerOptions{}) + if err != nil { + return errors.WithStack(err) + } + return nil + } +} + func (cr *containerReference) copyContent(dstPath string, files ...*FileEntry) common.Executor { return func(ctx context.Context) error { logger := common.Logger(ctx) diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go index a86a0090..429db613 100644 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -80,11 +80,6 @@ func (rc *RunContext) startJobContainer() common.Executor { bindModifiers = ":delegated" } - hostActionCache := os.Getenv("ACT_HOST_ACTION_CACHE") - if hostActionCache == "" { - hostActionCache = rc.ActionCacheDir() - } - envList = append(envList, fmt.Sprintf("%s=%s", "ACT_HOST_ACTION_CACHE", hostActionCache)) envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_TOOL_CACHE", "/toolcache")) rc.JobContainer = container.NewContainer(&container.NewContainerInput{ @@ -97,11 +92,11 @@ func (rc *RunContext) startJobContainer() common.Executor { Mounts: map[string]string{ name: "/github", "act-toolcache": "/toolcache", + "act-actions": "/actions", }, Binds: []string{ fmt.Sprintf("%s:%s%s", rc.Config.Workdir, "/github/workspace", bindModifiers), - fmt.Sprintf("%s:%s%s", hostActionCache, "/github/home/.cache/act", bindModifiers), fmt.Sprintf("%s:%s", "/var/run/docker.sock", "/var/run/docker.sock"), }, Stdout: logWriter, @@ -117,6 +112,10 @@ func (rc *RunContext) startJobContainer() common.Executor { Name: "workflow/event.json", Mode: 644, Body: rc.EventJSON, + }, &container.FileEntry{ + Name: "home/.act", + Mode: 644, + Body: "", }), )(ctx) } diff --git a/pkg/runner/step_context.go b/pkg/runner/step_context.go index a78211cf..7d2fd80e 100644 --- a/pkg/runner/step_context.go +++ b/pkg/runner/step_context.go @@ -65,7 +65,7 @@ func (sc *StepContext) Executor() common.Executor { } } - actionDir := fmt.Sprintf("%s/%s", rc.ActionCacheDir(), remoteAction.Repo) + actionDir := fmt.Sprintf("%s/%s", rc.ActionCacheDir(), strings.ReplaceAll(step.Uses, "/", "-")) return common.NewPipelineExecutor( common.NewGitCloneExecutor(common.NewGitCloneExecutorInput{ URL: remoteAction.CloneURL(), @@ -170,6 +170,7 @@ func (sc *StepContext) newStepContainer(ctx context.Context, image string, cmd [ Mounts: map[string]string{ rc.jobContainerName(): "/github", "act-toolcache": "/toolcache", + "act-actions": "/actions", }, Binds: []string{ fmt.Sprintf("%s:%s%s", rc.Config.Workdir, "/github/workspace", bindModifiers), @@ -234,16 +235,24 @@ func (sc *StepContext) runAction(actionDir string) common.Executor { actionName := "" containerActionDir := "." - if strings.HasPrefix(actionDir, rc.Config.Workdir) { - actionName = strings.TrimPrefix(actionDir, rc.Config.Workdir) + if step.Type() == model.StepTypeUsesActionLocal { + actionName = strings.TrimPrefix(strings.TrimPrefix(actionDir, rc.Config.Workdir), string(filepath.Separator)) containerActionDir = "/github/workspace" - } else if strings.HasPrefix(actionDir, rc.ActionCacheDir()) { - actionName = strings.TrimPrefix(actionDir, rc.ActionCacheDir()) - containerActionDir = "/github/home/.cache/act" + } else if step.Type() == model.StepTypeUsesActionRemote { + actionName = strings.TrimPrefix(strings.TrimPrefix(actionDir, rc.ActionCacheDir()), string(filepath.Separator)) + containerActionDir = "/actions" } + log.Debugf("actionDir=%s Workdir=%s ActionCacheDir=%s actionName=%s containerActionDir=%s", actionDir, rc.Config.Workdir, rc.ActionCacheDir(), actionName, containerActionDir) + switch action.Runs.Using { case model.ActionRunsUsingNode12: + if step.Type() == model.StepTypeUsesActionRemote { + err := rc.JobContainer.CopyDir(containerActionDir+string(filepath.Separator), actionDir)(ctx) + if err != nil { + return err + } + } return rc.execJobContainer([]string{"node", fmt.Sprintf("%s/%s/%s", containerActionDir, actionName, action.Runs.Main)}, sc.Env)(ctx) case model.ActionRunsUsingDocker: var prepImage common.Executor @@ -278,34 +287,6 @@ func (sc *StepContext) runAction(actionDir string) common.Executor { ).Finally( stepContainer.Remove().IfBool(!rc.Config.ReuseContainers), )(ctx) - - /* - case model.ActionRunsUsingNode12: - if strings.HasPrefix(actionDir, rc.Config.Workdir) { - containerSpec.Entrypoint = fmt.Sprintf("node /github/workspace/%s/%s", strings.TrimPrefix(actionDir, rc.Config.Workdir), action.Runs.Main) - } else if strings.HasPrefix(actionDir, rc.Tempdir) { - containerSpec.Entrypoint = fmt.Sprintf("node /github/home/%s/%s", strings.TrimPrefix(actionDir, rc.Tempdir), action.Runs.Main) - } - case model.ActionRunsUsingDocker: - if strings.HasPrefix(actionDir, rc.Config.Workdir) { - containerSpec.Name = rc.createStepContainerName(strings.TrimPrefix(actionDir, rc.Config.Workdir)) - } else if strings.HasPrefix(actionDir, rc.Tempdir) { - containerSpec.Name = rc.createStepContainerName(strings.TrimPrefix(actionDir, rc.Tempdir)) - } - containerSpec.Reuse = rc.Config.ReuseContainers - if strings.HasPrefix(action.Runs.Image, "docker://") { - containerSpec.Image = strings.TrimPrefix(action.Runs.Image, "docker://") - containerSpec.Entrypoint = strings.Join(action.Runs.Entrypoint, " ") - containerSpec.Args = strings.Join(action.Runs.Args, " ") - } else { - containerSpec.Image = fmt.Sprintf("%s:%s", containerSpec.Name, "latest") - contextDir := filepath.Join(actionDir, action.Runs.Main) - return container.NewDockerBuildExecutor(container.NewDockerBuildExecutorInput{ - ContextDir: contextDir, - ImageTag: containerSpec.Image, - })(ctx) - } - */ } return nil } From 1121f6e1329a2af1e1f1ab77a924cc9fe9c74905 Mon Sep 17 00:00:00 2001 From: Casey Lee Date: Mon, 24 Feb 2020 17:48:21 -0800 Subject: [PATCH 14/14] run with copy of workingdir --- cmd/input.go | 1 + cmd/root.go | 2 ++ pkg/container/docker_run.go | 5 ++++- pkg/runner/run_context.go | 13 +++++++++---- pkg/runner/runner.go | 1 + pkg/runner/step_context.go | 12 ++++++++---- pkg/runner/testdata/basic/push.yml | 1 + 7 files changed, 26 insertions(+), 9 deletions(-) diff --git a/cmd/input.go b/cmd/input.go index 63a64adc..3b1b34f9 100644 --- a/cmd/input.go +++ b/cmd/input.go @@ -11,6 +11,7 @@ type Input struct { workflowsPath string eventPath string reuseContainers bool + bindWorkdir bool secrets []string platforms []string dryrun bool diff --git a/cmd/root.go b/cmd/root.go index a22b10f7..efbc05c0 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -33,6 +33,7 @@ func Execute(ctx context.Context, version string) { rootCmd.Flags().StringArrayVarP(&input.secrets, "secret", "s", []string{}, "secret to make available to actions with optional value (e.g. -s mysecret=foo or -s mysecret)") rootCmd.Flags().StringArrayVarP(&input.platforms, "platform", "P", []string{}, "custom image to use per platform (e.g. -P ubuntu-18.04=nektos/act-environments-ubuntu:18.04)") rootCmd.Flags().BoolVarP(&input.reuseContainers, "reuse", "r", false, "reuse action containers to maintain state") + rootCmd.Flags().BoolVarP(&input.bindWorkdir, "bind", "b", false, "bind working directory to container, rather than copy") rootCmd.Flags().BoolVarP(&input.forcePull, "pull", "p", false, "pull docker image(s) if already present") rootCmd.Flags().StringVarP(&input.eventPath, "eventpath", "e", "", "path to event JSON file") rootCmd.PersistentFlags().StringVarP(&input.workflowsPath, "workflows", "W", "./.github/workflows/", "path to workflow files") @@ -97,6 +98,7 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str ForcePull: input.forcePull, ReuseContainers: input.reuseContainers, Workdir: input.Workdir(), + BindWorkdir: input.bindWorkdir, LogOutput: !input.noOutput, Secrets: newSecrets(input.secrets), Platforms: input.newPlatforms(), diff --git a/pkg/container/docker_run.go b/pkg/container/docker_run.go index f7ee069a..faf436b0 100644 --- a/pkg/container/docker_run.go +++ b/pkg/container/docker_run.go @@ -101,6 +101,7 @@ func (cr *containerReference) Copy(destPath string, files ...*FileEntry) common. func (cr *containerReference) CopyDir(destPath string, srcPath string) common.Executor { return common.NewPipelineExecutor( + common.NewInfoExecutor("%sdocker cp src=%s dst=%s", logPrefix, srcPath, destPath), cr.connect(), cr.find(), cr.copyDir(destPath, srcPath), @@ -310,13 +311,14 @@ func (cr *containerReference) copyDir(dstPath string, srcPath string) common.Exe } log.Debugf("Writing tarball %s from %s", tarFile.Name(), srcPath) defer tarFile.Close() - //defer os.Remove(tarFile.Name()) + defer os.Remove(tarFile.Name()) tw := tar.NewWriter(tarFile) srcPrefix := filepath.Dir(srcPath) if !strings.HasSuffix(srcPrefix, string(filepath.Separator)) { srcPrefix += string(filepath.Separator) } + log.Debugf("Stripping prefix:%s src:%s", srcPrefix, srcPath) err = filepath.Walk(srcPath, func(file string, fi os.FileInfo, err error) error { if err != nil { @@ -336,6 +338,7 @@ func (cr *containerReference) copyDir(dstPath string, srcPath string) common.Exe // update the name to correctly reflect the desired destination when untaring header.Name = strings.TrimPrefix(file, srcPrefix) + log.Debugf("%s -> %s", file, header.Name) // write the header if err := tw.WriteHeader(header); err != nil { diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go index 429db613..09d3ed2c 100644 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -82,6 +82,13 @@ func (rc *RunContext) startJobContainer() common.Executor { envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_TOOL_CACHE", "/toolcache")) + binds := []string{ + fmt.Sprintf("%s:%s", "/var/run/docker.sock", "/var/run/docker.sock"), + } + if rc.Config.BindWorkdir { + binds = append(binds, fmt.Sprintf("%s:%s%s", rc.Config.Workdir, "/github/workspace", bindModifiers)) + } + rc.JobContainer = container.NewContainer(&container.NewContainerInput{ Cmd: nil, Entrypoint: []string{"/usr/bin/tail", "-f", "/dev/null"}, @@ -95,10 +102,7 @@ func (rc *RunContext) startJobContainer() common.Executor { "act-actions": "/actions", }, - Binds: []string{ - fmt.Sprintf("%s:%s%s", rc.Config.Workdir, "/github/workspace", bindModifiers), - fmt.Sprintf("%s:%s", "/var/run/docker.sock", "/var/run/docker.sock"), - }, + Binds: binds, Stdout: logWriter, Stderr: logWriter, }) @@ -108,6 +112,7 @@ func (rc *RunContext) startJobContainer() common.Executor { rc.JobContainer.Remove().IfBool(!rc.Config.ReuseContainers), rc.JobContainer.Create(), rc.JobContainer.Start(false), + rc.JobContainer.CopyDir("/github/workspace", rc.Config.Workdir+"/.").IfBool(!rc.Config.BindWorkdir), rc.JobContainer.Copy("/github/", &container.FileEntry{ Name: "workflow/event.json", Mode: 644, diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index 6aa1eaf7..3befe0c8 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -18,6 +18,7 @@ type Runner interface { // Config contains the config for a new runner type Config struct { Workdir string // path to working directory + BindWorkdir bool // bind the workdir to the job container EventName string // name of event to run EventPath string // path to JSON file to use for event.json in containers ReuseContainers bool // reuse containers to maintain state diff --git a/pkg/runner/step_context.go b/pkg/runner/step_context.go index 7d2fd80e..cece632c 100644 --- a/pkg/runner/step_context.go +++ b/pkg/runner/step_context.go @@ -160,6 +160,13 @@ func (sc *StepContext) newStepContainer(ctx context.Context, image string, cmd [ envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_TOOL_CACHE", "/toolcache")) + binds := []string{ + fmt.Sprintf("%s:%s", "/var/run/docker.sock", "/var/run/docker.sock"), + } + if rc.Config.BindWorkdir { + binds = append(binds, fmt.Sprintf("%s:%s%s", rc.Config.Workdir, "/github/workspace", bindModifiers)) + } + stepContainer := container.NewContainer(&container.NewContainerInput{ Cmd: cmd, Entrypoint: entrypoint, @@ -172,10 +179,7 @@ func (sc *StepContext) newStepContainer(ctx context.Context, image string, cmd [ "act-toolcache": "/toolcache", "act-actions": "/actions", }, - Binds: []string{ - fmt.Sprintf("%s:%s%s", rc.Config.Workdir, "/github/workspace", bindModifiers), - fmt.Sprintf("%s:%s", "/var/run/docker.sock", "/var/run/docker.sock"), - }, + Binds: binds, Stdout: logWriter, Stderr: logWriter, }) diff --git a/pkg/runner/testdata/basic/push.yml b/pkg/runner/testdata/basic/push.yml index 30218acd..2ae1ea5e 100644 --- a/pkg/runner/testdata/basic/push.yml +++ b/pkg/runner/testdata/basic/push.yml @@ -5,6 +5,7 @@ jobs: check: runs-on: ubuntu-latest steps: + - run: ls - run: echo 'hello world' - run: echo ${GITHUB_SHA} >> /github/sha.txt - run: cat /github/sha.txt | grep ${GITHUB_SHA}