From 77eec5da2bb109167947a1110ae9b9c71e9f6f07 Mon Sep 17 00:00:00 2001 From: Srevin Saju Date: Sat, 14 Oct 2023 18:28:17 +0300 Subject: [PATCH 1/6] refactor: split stage run functions into smaller sub functions --- pkg/ci/stage_hooks.go | 4 +- pkg/ci/stage_run.go | 717 +++++++++++++++++++------------------- pkg/ci/stage_schema.go | 3 +- pkg/ci/stage_terminate.go | 93 +++++ tests/togomak.hcl | 8 +- 5 files changed, 465 insertions(+), 360 deletions(-) create mode 100644 pkg/ci/stage_terminate.go diff --git a/pkg/ci/stage_hooks.go b/pkg/ci/stage_hooks.go index 4db6d7d..01829ab 100644 --- a/pkg/ci/stage_hooks.go +++ b/pkg/ci/stage_hooks.go @@ -16,7 +16,7 @@ func (s *Stage) BeforeRun(ctx context.Context, opts ...runnable.Option) hcl.Diag for _, hook := range s.PreHook { diags = diags.Extend( - (&Stage{fmt.Sprintf("%s.pre", s.Identifier()), hook.Stage, nil}).Run(ctx, opts...), + (&Stage{fmt.Sprintf("%s.pre", s.Identifier()), nil, hook.Stage, nil}).Run(ctx, opts...), ) } return diags @@ -31,7 +31,7 @@ func (s *Stage) AfterRun(ctx context.Context, opts ...runnable.Option) hcl.Diagn for _, hook := range s.PostHook { diags = diags.Extend( - (&Stage{fmt.Sprintf("%s.pre", s.Identifier()), hook.Stage, nil}).Run(ctx, opts...), + (&Stage{fmt.Sprintf("%s.pre", s.Identifier()), nil, hook.Stage, nil}).Run(ctx, opts...), ) } return diags diff --git a/pkg/ci/stage_run.go b/pkg/ci/stage_run.go index 810d944..be9a274 100644 --- a/pkg/ci/stage_run.go +++ b/pkg/ci/stage_run.go @@ -2,6 +2,7 @@ package ci import ( "context" + "errors" "fmt" "github.com/alessio/shellescape" "github.com/docker/docker/api/types" @@ -26,7 +27,6 @@ import ( "path/filepath" "regexp" "strings" - "syscall" ) const TogomakParamEnvVarPrefix = "TOGOMAK__param__" @@ -347,6 +347,29 @@ func (s *Stage) Run(ctx context.Context, options ...runnable.Option) (diags hcl. diags = diags.Extend(d) logger.Debugf("finished expanding macros with %d errors", len(diags.Errs())) + if s.ForEach != nil { + global.EvalContextMutex.RLock() + forEachItems, d := s.ForEach.Value(evalCtx) + global.EvalContextMutex.RUnlock() + + diags = diags.Extend(d) + if !forEachItems.IsNull() { + if !forEachItems.CanIterateElements() { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "invalid type for for_each", + Detail: fmt.Sprintf("for_each must be a set or map of objects"), + }) + return diags + } + + forEachItems.ForEachElement(func(k cty.Value, v cty.Value) bool { + fmt.Println("ok!") + return true + }) + } + } + defer func() { logger.Debug("running post hooks") success := !diags.HasErrors() @@ -365,15 +388,8 @@ func (s *Stage) Run(ctx context.Context, options ...runnable.Option) (diags hcl. logger.Debug("finished running post hooks") }() - logger.Debugf("running pre hooks") - hookOpts := []runnable.Option{ - runnable.WithStatus(status), - runnable.WithHook(), - runnable.WithParent(runnable.ParentConfig{Name: s.Name, Id: s.Id}), - } - hookOpts = append(hookOpts, options...) - diags = diags.Extend(s.BeforeRun(ctx, hookOpts...)) - logger.Debugf("finished running pre hooks") + d = s.executePreHooks(ctx, status, options...) + diags = diags.Extend(d) paramsGo := map[string]cty.Value{} @@ -419,41 +435,250 @@ func (s *Stage) Run(ctx context.Context, options ...runnable.Option) (diags hcl. } } } - evalCtx.Variables[ParamBlock] = cty.ObjectVal(paramsGo) - logger.Debug("evaluating script value") - global.EvalContextMutex.RLock() - script, d := s.Script.Value(evalCtx) - global.EvalContextMutex.RUnlock() + environment, d := s.parseEnvironmentVariables(evalCtx) + diags = diags.Extend(d) + if diags.HasErrors() { + return diags + } - if d.HasErrors() && cfg.Behavior.DryRun { - script = cty.StringVal(ui.Italic(ui.Yellow("(will be evaluated later)"))) + envStrings := s.processEnvironmentVariables(environment, cfg, tmpDir, paramsGo) + + cmd, d := s.parseExecCommand(ctx, evalCtx, cfg, envStrings) + diags = diags.Extend(d) + if diags.HasErrors() { + return diags + } + logger.Trace("command parsed") + logger.Tracef("script: %.30s... ", cmd.String()) + + if s.Container == nil { + s.process = cmd + logger.Trace("running command:", cmd.String()) + if !cfg.Behavior.DryRun { + err = cmd.Run() + + if err != nil && err.Error() == "signal: terminated" && s.Terminated() { + logger.Warnf("command terminated with signal: %s", cmd.ProcessState.String()) + err = nil + } + } else { + fmt.Println(cmd.String()) + } } else { + d := s.executeDocker(ctx, evalCtx, cmd, cfg) diags = diags.Extend(d) } - global.EvalContextMutex.RLock() - shellRaw, d := s.Shell.Value(evalCtx) - global.EvalContextMutex.RUnlock() + if err != nil { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("failed to run command (%s)", s.Identifier()), + Detail: err.Error(), + }) + } - shell := "" - if d.HasErrors() { + return diags +} + +func (s *Stage) executeDocker(ctx context.Context, evalCtx *hcl.EvalContext, cmd *exec.Cmd, cfg *runnable.Config) hcl.Diagnostics { + var diags hcl.Diagnostics + logger := s.Logger().WithField("🐳", "") + + image, d := s.hclImage(evalCtx) + diags = diags.Extend(d) + + // begin entrypoint evaluation + entrypoint, d := s.hclEndpoint(evalCtx) + diags = diags.Extend(d) + + if diags.HasErrors() { + return diags + } + + cli, err := dockerClient.NewClientWithOpts(dockerClient.FromEnv, dockerClient.WithAPIVersionNegotiation()) + if err != nil { + return diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "could not create docker client", + Detail: err.Error(), + Subject: s.Container.Image.Range().Ptr(), + EvalContext: evalCtx, + }) + } + defer x.Must(cli.Close()) + + // check if image exists + logger.Debugf("checking if image %s exists", image) + _, _, err = cli.ImageInspectWithRaw(ctx, image) + if err != nil { + logger.Infof("image %s does not exist, pulling...", image) + reader, err := cli.ImagePull(ctx, image, types.ImagePullOptions{}) + if err != nil { + return diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "could not pull image", + Detail: err.Error(), + Subject: s.Container.Image.Range().Ptr(), + EvalContext: evalCtx, + }) + } + + pb := ui.NewDockerProgressWriter(reader, logger.Writer(), fmt.Sprintf("pulling image %s", image)) + defer pb.Close() + defer reader.Close() + io.Copy(pb, reader) + } + + logger.Trace("parsing container arguments") + binds := []string{ + fmt.Sprintf("%s:/workspace", cmd.Dir), + } + + logger.Trace("parsing container volumes") + for _, m := range s.Container.Volumes { + global.EvalContextMutex.RLock() + source, d := m.Source.Value(evalCtx) + global.EvalContextMutex.RUnlock() diags = diags.Extend(d) - } else { - if shellRaw.IsNull() { - shell = "bash" - } else { - shell = shellRaw.AsString() + + global.EvalContextMutex.RLock() + dest, d := m.Destination.Value(evalCtx) + global.EvalContextMutex.RUnlock() + diags = diags.Extend(d) + if diags.HasErrors() { + continue } + binds = append(binds, fmt.Sprintf("%s:%s", source.AsString(), dest.AsString())) + } + logger.Tracef("%d diagnostic(s) after parsing container volumes", len(diags.Errs())) + if diags.HasErrors() { + return diags } - global.EvalContextMutex.RLock() - args, d := s.Args.Value(evalCtx) - global.EvalContextMutex.RUnlock() + logger.Trace("dry run check") + if cfg.Behavior.DryRun { + fmt.Println(ui.Blue("docker:run.image"), ui.Green(image)) + fmt.Println(ui.Blue("docker:run.workdir"), ui.Green("/workspace")) + fmt.Println(ui.Blue("docker:run.volume"), ui.Green(cmd.Dir+":/workspace")) + fmt.Println(ui.Blue("docker:run.stdin"), ui.Green(s.Container.Stdin)) + fmt.Println(ui.Blue("docker:run.args"), ui.Green(cmd.String())) + return diags + } + + logger.Trace("parsing container ports") + exposedPorts, bindings, d := s.Container.Ports.Nat(evalCtx) diags = diags.Extend(d) + if diags.HasErrors() { + return diags + } + + logger.Trace("creating container") + resp, err := cli.ContainerCreate(ctx, &dockerContainer.Config{ + Image: image, + Cmd: cmd.Args, + WorkingDir: "/workspace", + Volumes: map[string]struct{}{ + "/workspace": {}, + }, + Tty: true, + AttachStdout: true, + AttachStderr: true, + AttachStdin: s.Container.Stdin, + OpenStdin: s.Container.Stdin, + StdinOnce: s.Container.Stdin, + Entrypoint: entrypoint, + Env: cmd.Env, + ExposedPorts: exposedPorts, + // User: s.Container.User, + }, &dockerContainer.HostConfig{ + Binds: binds, + PortBindings: bindings, + }, nil, nil, "") + if err != nil { + return diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "could not create container", + Detail: err.Error(), + Subject: s.Container.Image.Range().Ptr(), + EvalContext: evalCtx, + }) + } + + logger.Trace("starting container") + if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil { + return diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "could not start container", + Detail: err.Error(), + Subject: s.Container.Image.Range().Ptr(), + }) + } + s.ContainerId = resp.ID + + logger.Trace("getting container metadata for log retrieval") + container, err := cli.ContainerInspect(ctx, resp.ID) + if err != nil { + panic(err) + } + + logger.Trace("getting container logs") + responseBody, err := cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ + ShowStdout: true, ShowStderr: true, + Follow: true, + }) + if err != nil { + return diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "could not get container logs", + Detail: err.Error(), + Subject: s.Container.Image.Range().Ptr(), + }) + } + defer responseBody.Close() + + logger.Tracef("copying container logs on container: %s", resp.ID) + if container.Config.Tty { + _, err = io.Copy(logger.Writer(), responseBody) + } else { + _, err = stdcopy.StdCopy(logger.Writer(), logger.WriterLevel(logrus.WarnLevel), responseBody) + } + + logger.Trace("waiting for container to finish") + if err != nil && err != io.EOF { + if errors.Is(err, context.Canceled) { + return diags + } + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "failed to copy container logs", + Detail: err.Error(), + Subject: s.Container.Image.Range().Ptr(), + }) + } + + logger.Tracef("removing container with id: %s", resp.ID) + err = cli.ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{ + RemoveVolumes: true, + }) + if err != nil { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "failed to remove container", + Detail: err.Error(), + Subject: s.Container.Image.Range().Ptr(), + }) + return diags + } + + logger.Tracef("%d diagnostic(s) after removing container", len(diags.Errs())) + return diags +} - logger.Debug("evaluating environment variables") +func (s *Stage) parseEnvironmentVariables(evalCtx *hcl.EvalContext) (map[string]cty.Value, hcl.Diagnostics) { + var diags hcl.Diagnostics + s.Logger().Debug("evaluating environment variables") var environment map[string]cty.Value environment = make(map[string]cty.Value) for _, env := range s.Environment { @@ -482,38 +707,88 @@ func (s *Stage) Run(ctx context.Context, options ...runnable.Option) (diags hcl. environment[env.Name] = v } } + return environment, diags +} - logger.Debugf("%d diagnostics so far", len(diags.Errs())) - if diags.HasErrors() { - return diags +func (s *Stage) parseExecCommand(ctx context.Context, evalCtx *hcl.EvalContext, cfg *runnable.Config, envStrings []string) (*exec.Cmd, hcl.Diagnostics) { + var diags hcl.Diagnostics + + s.Logger().Trace("evaluating script value") + global.EvalContextMutex.RLock() + script, d := s.Script.Value(evalCtx) + global.EvalContextMutex.RUnlock() + + if d.HasErrors() && cfg.Behavior.DryRun { + script = cty.StringVal(ui.Italic(ui.Yellow("(will be evaluated later)"))) + } else { + diags = diags.Extend(d) } - envStrings := make([]string, len(environment)) - envCounter := 0 - for k, v := range environment { - envParsed := fmt.Sprintf("%s=%s", k, v.AsString()) - if cfg.Behavior.DryRun { - fmt.Println(ui.Blue("export"), envParsed) + s.Logger().Trace("evaluating shell value") + global.EvalContextMutex.RLock() + shellRaw, d := s.Shell.Value(evalCtx) + global.EvalContextMutex.RUnlock() + + shell := "" + if d.HasErrors() { + diags = diags.Extend(d) + } else { + if shellRaw.IsNull() { + shell = "bash" + } else { + shell = shellRaw.AsString() } + } - envStrings[envCounter] = envParsed - envCounter = envCounter + 1 + s.Logger().Trace("evaluating args value") + global.EvalContextMutex.RLock() + args, d := s.Args.Value(evalCtx) + global.EvalContextMutex.RUnlock() + diags = diags.Extend(d) + + cmdHcl, d := s.parseCommand(evalCtx, shell, script, args) + diags = diags.Extend(d) + if diags.HasErrors() { + return nil, diags } - togomakEnvExport := fmt.Sprintf("%s=%s", meta.OutputEnvVar, filepath.Join(tmpDir, meta.OutputEnvFile)) - logger.Tracef("exporting %s", togomakEnvExport) - envStrings = append(envStrings, togomakEnvExport) - if s.Use != nil && s.Use.Parameters != nil { - for k, v := range paramsGo { - envParsed := fmt.Sprintf("%s%s=%s", TogomakParamEnvVarPrefix, k, v.AsString()) - if cfg.Behavior.DryRun { - fmt.Println(ui.Blue("export"), envParsed) - } + dir := cfg.Paths.Cwd - envStrings = append(envStrings, envParsed) + global.EvalContextMutex.RLock() + dirParsed, d := s.Dir.Value(evalCtx) + global.EvalContextMutex.RUnlock() + + if d.HasErrors() { + diags = diags.Extend(d) + } else { + if !dirParsed.IsNull() && dirParsed.AsString() != "" { + dir = dirParsed.AsString() + } + if !filepath.IsAbs(dir) { + dir = filepath.Join(cfg.Paths.Cwd, dir) + } + if cfg.Behavior.DryRun { + fmt.Println(ui.Blue("cd"), dir) } } + cmd := exec.CommandContext(ctx, cmdHcl.command, cmdHcl.args...) + cmd.Stdout = s.Logger().Writer() + cmd.Stderr = s.Logger().WriterLevel(logrus.WarnLevel) + cmd.Env = append(os.Environ(), envStrings...) + cmd.Dir = dir + return cmd, diags +} + +type command struct { + args []string + command string + + isEmpty bool +} + +func (s *Stage) parseCommand(evalCtx *hcl.EvalContext, shell string, script cty.Value, args cty.Value) (command, hcl.Diagnostics) { + var diags hcl.Diagnostics runArgs := make([]string, 0) runCommand := shell @@ -537,246 +812,63 @@ func (s *Stage) Run(ctx context.Context, options ...runnable.Option) (diags hcl. } } else if s.Container == nil { // if the container is not null, we may rely on internal args or entrypoint scripts - return diags.Append(&hcl.Diagnostic{ + diags = diags.Append(&hcl.Diagnostic{ Severity: hcl.DiagError, Summary: "No commands specified", Detail: "Either script or args must be specified", Subject: s.Script.Range().Ptr(), EvalContext: evalCtx, }) - } else { emptyCommands = true } - dir := cfg.Paths.Cwd - - global.EvalContextMutex.RLock() - dirParsed, d := s.Dir.Value(evalCtx) - global.EvalContextMutex.RUnlock() + return command{ + args: runArgs, + command: runCommand, + isEmpty: emptyCommands, + }, diags +} - if d.HasErrors() { - diags = diags.Extend(d) - } else { - if !dirParsed.IsNull() && dirParsed.AsString() != "" { - dir = dirParsed.AsString() - } - if !filepath.IsAbs(dir) { - dir = filepath.Join(cfg.Paths.Cwd, dir) - } +func (s *Stage) processEnvironmentVariables(environment map[string]cty.Value, cfg *runnable.Config, tmpDir string, paramsGo map[string]cty.Value) []string { + envStrings := make([]string, len(environment)) + envCounter := 0 + for k, v := range environment { + envParsed := fmt.Sprintf("%s=%s", k, v.AsString()) if cfg.Behavior.DryRun { - fmt.Println(ui.Blue("cd"), dir) - } - } - - cmd := exec.CommandContext(ctx, runCommand, runArgs...) - cmd.Stdout = logger.Writer() - cmd.Stderr = logger.WriterLevel(logrus.WarnLevel) - cmd.Env = append(os.Environ(), envStrings...) - cmd.Dir = dir - - if s.Container == nil { - s.process = cmd - logger.Trace("running command:", cmd.String()) - if !cfg.Behavior.DryRun { - err = cmd.Run() - - if err != nil && err.Error() == "signal: terminated" && s.Terminated() { - logger.Warnf("command terminated with signal: %s", cmd.ProcessState.String()) - err = nil - } - } else { - fmt.Println(cmd.String()) - } - } else { - logger := logger.WithField("🐳", "") - - image, d := s.hclImage(evalCtx) - diags = diags.Extend(d) - - // begin entrypoint evaluation - entrypoint, d := s.hclEndpoint(evalCtx) - diags = diags.Extend(d) - - if diags.HasErrors() { - return diags - } - - cli, err := dockerClient.NewClientWithOpts(dockerClient.FromEnv, dockerClient.WithAPIVersionNegotiation()) - if err != nil { - return diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "could not create docker client", - Detail: err.Error(), - Subject: s.Container.Image.Range().Ptr(), - EvalContext: evalCtx, - }) - } - defer cli.Close() - // check if image exists - logger.Debugf("checking if image %s exists", image) - _, _, err = cli.ImageInspectWithRaw(ctx, image) - if err != nil { - logger.Infof("image %s does not exist, pulling...", image) - reader, err := cli.ImagePull(ctx, image, types.ImagePullOptions{}) - if err != nil { - return diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "could not pull image", - Detail: err.Error(), - Subject: s.Container.Image.Range().Ptr(), - EvalContext: evalCtx, - }) - } - - pb := ui.NewDockerProgressWriter(reader, logger.Writer(), fmt.Sprintf("pulling image %s", image)) - defer pb.Close() - defer reader.Close() - io.Copy(pb, reader) - - } - - var containerArgs []string - if !emptyCommands { - containerArgs = cmd.Args - } - binds := []string{ - fmt.Sprintf("%s:/workspace", cmd.Dir), - } - - for _, m := range s.Container.Volumes { - global.EvalContextMutex.RLock() - source, d := m.Source.Value(evalCtx) - global.EvalContextMutex.RUnlock() - diags = diags.Extend(d) - - global.EvalContextMutex.RLock() - dest, d := m.Destination.Value(evalCtx) - global.EvalContextMutex.RUnlock() - diags = diags.Extend(d) - if diags.HasErrors() { - continue - } - binds = append(binds, fmt.Sprintf("%s:%s", source.AsString(), dest.AsString())) - } - if diags.HasErrors() { - return diags + fmt.Println(ui.Blue("export"), envParsed) } - if !cfg.Behavior.DryRun { - exposedPorts, bindings, d := s.Container.Ports.Nat(evalCtx) - diags = diags.Extend(d) - if diags.HasErrors() { - return diags - } - - resp, err := cli.ContainerCreate(ctx, &dockerContainer.Config{ - Image: image, - Cmd: containerArgs, - WorkingDir: "/workspace", - Volumes: map[string]struct{}{ - "/workspace": {}, - }, - Tty: true, - AttachStdout: true, - AttachStderr: true, - AttachStdin: s.Container.Stdin, - OpenStdin: s.Container.Stdin, - StdinOnce: s.Container.Stdin, - Entrypoint: entrypoint, - Env: envStrings, - ExposedPorts: exposedPorts, - // User: s.Container.User, - }, &dockerContainer.HostConfig{ - Binds: binds, - PortBindings: bindings, - }, nil, nil, "") - if err != nil { - return diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "could not create container", - Detail: err.Error(), - Subject: s.Container.Image.Range().Ptr(), - EvalContext: evalCtx, - }) - } - - if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil { - return diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "could not start container", - Detail: err.Error(), - Subject: s.Container.Image.Range().Ptr(), - }) - } - s.ContainerId = resp.ID - - container, err := cli.ContainerInspect(ctx, resp.ID) - if err != nil { - panic(err) - } - - responseBody, err := cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ - ShowStdout: true, ShowStderr: true, - Follow: true, - }) - if err != nil { - return diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "could not get container logs", - Detail: err.Error(), - Subject: s.Container.Image.Range().Ptr(), - }) - } - defer responseBody.Close() + envStrings[envCounter] = envParsed + envCounter = envCounter + 1 + } + togomakEnvExport := fmt.Sprintf("%s=%s", meta.OutputEnvVar, filepath.Join(tmpDir, meta.OutputEnvFile)) + s.Logger().Tracef("exporting %s", togomakEnvExport) + envStrings = append(envStrings, togomakEnvExport) - if container.Config.Tty { - _, err = io.Copy(logger.Writer(), responseBody) - } else { - _, err = stdcopy.StdCopy(logger.Writer(), logger.WriterLevel(logrus.WarnLevel), responseBody) - } - if err != nil && err != io.EOF { - if err == context.Canceled { - return diags - } - diags = diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "failed to copy container logs", - Detail: err.Error(), - Subject: s.Container.Image.Range().Ptr(), - }) - } - err = cli.ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{ - RemoveVolumes: true, - }) - if err != nil { - diags = diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "failed to remove container", - Detail: err.Error(), - Subject: s.Container.Image.Range().Ptr(), - }) - return diags + if s.Use != nil && s.Use.Parameters != nil { + for k, v := range paramsGo { + envParsed := fmt.Sprintf("%s%s=%s", TogomakParamEnvVarPrefix, k, v.AsString()) + if cfg.Behavior.DryRun { + fmt.Println(ui.Blue("export"), envParsed) } - } else { - fmt.Println(ui.Blue("docker:run.image"), ui.Green(image)) - fmt.Println(ui.Blue("docker:run.workdir"), ui.Green("/workspace")) - fmt.Println(ui.Blue("docker:run.volume"), ui.Green(cmd.Dir+":/workspace")) - fmt.Println(ui.Blue("docker:run.env"), ui.Green(strings.Join(envStrings, " "))) - fmt.Println(ui.Blue("docker:run.stdin"), ui.Green(s.Container.Stdin)) - fmt.Println(ui.Blue("docker:run.args"), ui.Green(strings.Join(containerArgs, " "))) + envStrings = append(envStrings, envParsed) } - } + return envStrings +} - if err != nil { - diags = diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: fmt.Sprintf("failed to run command (%s)", s.Identifier()), - Detail: err.Error(), - }) +func (s *Stage) executePreHooks(ctx context.Context, status runnable.StatusType, options ...runnable.Option) hcl.Diagnostics { + var diags hcl.Diagnostics + s.Logger().Debugf("running pre hooks") + hookOpts := []runnable.Option{ + runnable.WithStatus(status), + runnable.WithHook(), + runnable.WithParent(runnable.ParentConfig{Name: s.Name, Id: s.Id}), } - + hookOpts = append(hookOpts, options...) + diags = diags.Extend(s.BeforeRun(ctx, hookOpts...)) + s.Logger().Debugf("finished running pre hooks") return diags } @@ -889,84 +981,3 @@ func (s *Stage) CanRun(ctx context.Context, options ...runnable.Option) (ok bool func dockerContainerSourceFmt(containerId string) string { return fmt.Sprintf("docker: container=%s", containerId) } - -func (s *Stage) Terminate(safe bool) hcl.Diagnostics { - s.Logger().Debug("terminating stage") - ctx := context.Background() - var diags hcl.Diagnostics - if safe { - s.terminated = true - } - - defer func() { - diags = diags.Extend(s.AfterRun( - ctx, - runnable.WithHook(), - runnable.WithStatus(runnable.StatusTerminated), - runnable.WithParent(runnable.ParentConfig{Name: s.Name, Id: s.Id}), - )) - }() - - if s.Container != nil && s.ContainerId != "" { - - cli, err := dockerClient.NewClientWithOpts(dockerClient.FromEnv, dockerClient.WithAPIVersionNegotiation()) - if err != nil { - diags = diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "failed to create docker client", - Detail: fmt.Sprintf("%s: %s", dockerContainerSourceFmt(s.ContainerId), err.Error()), - }) - } - s.Logger().Debug("stopping container") - err = cli.ContainerStop(ctx, s.ContainerId, dockerContainer.StopOptions{}) - if err != nil { - diags = diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "failed to stop container", - Detail: fmt.Sprintf("%s: %s", dockerContainerSourceFmt(s.ContainerId), err.Error()), - }) - } - s.Logger().Debug("removing container") - err = cli.ContainerRemove(ctx, s.ContainerId, types.ContainerRemoveOptions{ - RemoveVolumes: true, - }) - s.Logger().Debug("removed container") - - } else if s.process != nil && s.process.Process != nil { - if s.process.ProcessState != nil { - if s.process.ProcessState.Exited() { - return diags - } - } - err := s.process.Process.Signal(syscall.SIGTERM) - if err != nil { - diags = diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "failed to terminate process", - Detail: err.Error(), - }) - } - } - s.Logger().Debug("terminated stage") - - return diags -} - -func (s *Stage) Kill() hcl.Diagnostics { - diags := s.Terminate(false) - if s.process != nil && !s.process.ProcessState.Exited() { - err := s.process.Process.Kill() - if err != nil { - diags = diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "couldn't kill stage", - Detail: err.Error(), - }) - } - } - return diags -} - -func (s *Stage) Terminated() bool { - return s.terminated -} diff --git a/pkg/ci/stage_schema.go b/pkg/ci/stage_schema.go index ac5c9c8..32a847f 100644 --- a/pkg/ci/stage_schema.go +++ b/pkg/ci/stage_schema.go @@ -131,7 +131,8 @@ type StagePreHook struct { // scripts, docker containers, etc. A Stage receives all properties as that of CoreStage // along with an Id which is used by Stages to uniquely identify a stage. type Stage struct { - Id string `hcl:"id,label" json:"id" expr:"id"` + Id string `hcl:"id,label" json:"id" expr:"id"` + ForEach hcl.Expression `hcl:"for_each,optional" json:"for_each"` CoreStage `hcl:",remain"` // Lifecycle rules tell the termination policy of a daemon stage diff --git a/pkg/ci/stage_terminate.go b/pkg/ci/stage_terminate.go new file mode 100644 index 0000000..489b90e --- /dev/null +++ b/pkg/ci/stage_terminate.go @@ -0,0 +1,93 @@ +package ci + +import ( + "context" + "fmt" + "github.com/docker/docker/api/types" + dockerContainer "github.com/docker/docker/api/types/container" + dockerClient "github.com/docker/docker/client" + "github.com/hashicorp/hcl/v2" + "github.com/srevinsaju/togomak/v1/pkg/runnable" + "syscall" +) + +func (s *Stage) Terminate(safe bool) hcl.Diagnostics { + s.Logger().Debug("terminating stage") + ctx := context.Background() + var diags hcl.Diagnostics + if safe { + s.terminated = true + } + + defer func() { + diags = diags.Extend(s.AfterRun( + ctx, + runnable.WithHook(), + runnable.WithStatus(runnable.StatusTerminated), + runnable.WithParent(runnable.ParentConfig{Name: s.Name, Id: s.Id}), + )) + }() + + if s.Container != nil && s.ContainerId != "" { + + cli, err := dockerClient.NewClientWithOpts(dockerClient.FromEnv, dockerClient.WithAPIVersionNegotiation()) + if err != nil { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "failed to create docker client", + Detail: fmt.Sprintf("%s: %s", dockerContainerSourceFmt(s.ContainerId), err.Error()), + }) + } + s.Logger().Debug("stopping container") + err = cli.ContainerStop(ctx, s.ContainerId, dockerContainer.StopOptions{}) + if err != nil { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "failed to stop container", + Detail: fmt.Sprintf("%s: %s", dockerContainerSourceFmt(s.ContainerId), err.Error()), + }) + } + s.Logger().Debug("removing container") + err = cli.ContainerRemove(ctx, s.ContainerId, types.ContainerRemoveOptions{ + RemoveVolumes: true, + }) + s.Logger().Debug("removed container") + + } else if s.process != nil && s.process.Process != nil { + if s.process.ProcessState != nil { + if s.process.ProcessState.Exited() { + return diags + } + } + err := s.process.Process.Signal(syscall.SIGTERM) + if err != nil { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "failed to terminate process", + Detail: err.Error(), + }) + } + } + s.Logger().Debug("terminated stage") + + return diags +} + +func (s *Stage) Kill() hcl.Diagnostics { + diags := s.Terminate(false) + if s.process != nil && !s.process.ProcessState.Exited() { + err := s.process.Process.Kill() + if err != nil { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "couldn't kill stage", + Detail: err.Error(), + }) + } + } + return diags +} + +func (s *Stage) Terminated() bool { + return s.terminated +} diff --git a/tests/togomak.hcl b/tests/togomak.hcl index 6434eae..0fd43cb 100644 --- a/tests/togomak.hcl +++ b/tests/togomak.hcl @@ -32,9 +32,9 @@ stage "integration_tests" { ls ../examples for i in ../examples/*; do echo ${ansi.fg.green}$i${ansi.reset} - ./togomak_coverage -C "$i" --ci -v - ./togomak_coverage -C "$i" --ci -v root - ./togomak_coverage -C "$i" --ci -v -n + ./togomak_coverage -C "$i" --ci -v -v -v + ./togomak_coverage -C "$i" --ci -v -v -v root + ./togomak_coverage -C "$i" --ci -v -v -v -n done ./togomak_coverage cache clean --recursive ./togomak_coverage fmt --check --recursive @@ -42,7 +42,7 @@ stage "integration_tests" { for i in tests/failing/*; do set +e echo ${ansi.fg.green}$i${ansi.reset} - ./togomak_coverage -C "$i" --ci -v + ./togomak_coverage -C "$i" --ci -v -v -v result=$? if [ $result -eq 0 ]; then set -e From 602c56985ea0b0779fc194c1a9e5555bdf293347 Mon Sep 17 00:00:00 2001 From: Srevin Saju Date: Sat, 14 Oct 2023 22:31:25 +0300 Subject: [PATCH 2/6] feat: update logger and logging statements --- go.mod | 17 +++++------------ go.sum | 17 ++--------------- pkg/ci/stage_hooks.go | 4 ++-- pkg/ci/stage_run.go | 14 +++++++------- pkg/handler/handler.go | 6 +++--- pkg/log/logger.go | 1 + pkg/orchestra/orchestra.go | 4 ++-- pkg/orchestra/outputs.go | 2 +- pkg/orchestra/run.go | 5 +++-- pkg/ui/colors.go | 1 + 10 files changed, 27 insertions(+), 44 deletions(-) create mode 100644 pkg/log/logger.go diff --git a/go.mod b/go.mod index 217dc3a..3ab9951 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,6 @@ require ( github.com/bcicen/jstream v1.0.1 github.com/bmatcuk/doublestar v1.1.5 github.com/creack/pty v1.1.18 - github.com/docker/cli v24.0.6+incompatible github.com/docker/docker v24.0.2+incompatible github.com/docker/go-connections v0.4.0 github.com/fatih/color v1.15.0 @@ -33,6 +32,8 @@ require ( github.com/zclconf/go-cty-yaml v1.0.3 golang.org/x/crypto v0.11.0 golang.org/x/text v0.11.0 + google.golang.org/grpc v1.50.1 + google.golang.org/protobuf v1.28.1 ) require ( @@ -45,7 +46,6 @@ require ( github.com/aws/aws-sdk-go v1.44.122 // indirect github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect github.com/googleapis/gax-go/v2 v2.6.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect @@ -54,20 +54,13 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/compress v1.15.14 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/ulikunitz/xz v0.5.11 // indirect - github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect - github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect - github.com/xeipuuv/gojsonschema v1.2.0 // indirect go.opencensus.io v0.23.0 // indirect golang.org/x/oauth2 v0.3.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/api v0.100.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71 // indirect - google.golang.org/grpc v1.50.1 // indirect - google.golang.org/protobuf v1.28.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect ) require ( @@ -81,7 +74,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cloudflare/circl v1.3.3 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect - github.com/davecgh/go-spew v1.1.1 + github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/djherbis/buffer v1.2.0 // indirect github.com/djherbis/nio/v3 v3.0.1 // indirect @@ -143,6 +136,6 @@ require ( gotest.tools/v3 v3.4.0 // indirect ) -replace github.com/sirupsen/logrus v1.9.0 => github.com/srevinsaju/logrus v1.10.2-0.20230616201049-4e5a7636a82f +replace github.com/sirupsen/logrus v1.9.0 => github.com/srevinsaju/logrus v1.10.3-0.20231014192105-f54cbb8bc619 -replace github.com/sirupsen/logrus v1.9.2 => github.com/srevinsaju/logrus v1.10.2-0.20230616201049-4e5a7636a82f +replace github.com/sirupsen/logrus v1.9.2 => github.com/srevinsaju/logrus v1.10.3-0.20231014192105-f54cbb8bc619 diff --git a/go.sum b/go.sum index 952b616..528372f 100644 --- a/go.sum +++ b/go.sum @@ -266,8 +266,6 @@ github.com/djherbis/buffer v1.2.0 h1:PH5Dd2ss0C7CRRhQCZ2u7MssF+No9ide8Ye71nPHcrQ github.com/djherbis/buffer v1.2.0/go.mod h1:fjnebbZjCUpPinBRD+TDwXSOeNQ7fPQWLfGQqiAiUyE= github.com/djherbis/nio/v3 v3.0.1 h1:6wxhnuppteMa6RHA4L81Dq7ThkZH8SwnDzXDYy95vB4= github.com/djherbis/nio/v3 v3.0.1/go.mod h1:Ng4h80pbZFMla1yKzm61cF0tqqilXZYrogmWgZxOcmg= -github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY= -github.com/docker/cli v24.0.6+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v24.0.2+incompatible h1:eATx+oLz9WdNVkQrr0qjQ8HvRJ4bOOxfzEo8R+dA3cg= @@ -406,8 +404,6 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b h1:8htHrh2bw9c7Idkb7YNac+ZpTqLMjRpI+FWu51ltaQc= github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -520,8 +516,6 @@ github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJ github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= @@ -581,8 +575,8 @@ github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:s github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= -github.com/srevinsaju/logrus v1.10.2-0.20230616201049-4e5a7636a82f h1:fqDRPqbyG4oiL4x03rD5xxnOPTuKScc1Azs3e6xVFb4= -github.com/srevinsaju/logrus v1.10.2-0.20230616201049-4e5a7636a82f/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/srevinsaju/logrus v1.10.3-0.20231014192105-f54cbb8bc619 h1:YDqV1sOerHY5It8hZRqRpRWSKjaKSmbhxp5cjvMtsGY= +github.com/srevinsaju/logrus v1.10.3-0.20231014192105-f54cbb8bc619/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -608,13 +602,6 @@ github.com/urfave/cli/v2 v2.25.5 h1:d0NIAyhh5shGscroL7ek/Ya9QYQE0KNabJgiUinIQkc= github.com/urfave/cli/v2 v2.25.5/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= -github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/pkg/ci/stage_hooks.go b/pkg/ci/stage_hooks.go index 01829ab..fb70d3a 100644 --- a/pkg/ci/stage_hooks.go +++ b/pkg/ci/stage_hooks.go @@ -16,7 +16,7 @@ func (s *Stage) BeforeRun(ctx context.Context, opts ...runnable.Option) hcl.Diag for _, hook := range s.PreHook { diags = diags.Extend( - (&Stage{fmt.Sprintf("%s.pre", s.Identifier()), nil, hook.Stage, nil}).Run(ctx, opts...), + (&Stage{fmt.Sprintf("%s.pre", s.Id), nil, hook.Stage, nil}).Run(ctx, opts...), ) } return diags @@ -31,7 +31,7 @@ func (s *Stage) AfterRun(ctx context.Context, opts ...runnable.Option) hcl.Diagn for _, hook := range s.PostHook { diags = diags.Extend( - (&Stage{fmt.Sprintf("%s.pre", s.Identifier()), nil, hook.Stage, nil}).Run(ctx, opts...), + (&Stage{fmt.Sprintf("%s.pre", s.Id), nil, hook.Stage, nil}).Run(ctx, opts...), ) } return diags diff --git a/pkg/ci/stage_run.go b/pkg/ci/stage_run.go index be9a274..02b59ad 100644 --- a/pkg/ci/stage_run.go +++ b/pkg/ci/stage_run.go @@ -40,14 +40,14 @@ func (s *Stage) Prepare(ctx context.Context, skip bool, overridden bool) hcl.Dia var id string if !skip { - id = ui.Blue(s.Id) + id = "" } else { - id = fmt.Sprintf("%s %s", ui.Yellow(s.Id), ui.Grey("(skipped)")) + id = fmt.Sprintf("%s", ui.Grey("skipped")) } if overridden { - id = fmt.Sprintf("%s %s", id, ui.Bold("(overriden)")) + id = fmt.Sprintf("%s", ui.Blue("overridden")) } - logger.Infof("[%s] %s", ui.Plus, id) + logger.Infof("%s", id) return nil } @@ -60,7 +60,7 @@ func (s *Stage) expandMacros(ctx context.Context, opts ...runnable.Option) (*Sta return s, nil } hclContext := global.HclEvalContext() - logger := s.Logger().WithField(MacroBlock, true) + logger := s.Logger() pipe := ctx.Value(c.TogomakContextPipeline).(*Pipeline) tmpDir := global.TempDir() @@ -455,7 +455,7 @@ func (s *Stage) Run(ctx context.Context, options ...runnable.Option) (diags hcl. if s.Container == nil { s.process = cmd - logger.Trace("running command:", cmd.String()) + logger.Tracef("running command: %.30s...", cmd.String()) if !cfg.Behavior.DryRun { err = cmd.Run() @@ -484,7 +484,7 @@ func (s *Stage) Run(ctx context.Context, options ...runnable.Option) (diags hcl. func (s *Stage) executeDocker(ctx context.Context, evalCtx *hcl.EvalContext, cmd *exec.Cmd, cfg *runnable.Config) hcl.Diagnostics { var diags hcl.Diagnostics - logger := s.Logger().WithField("🐳", "") + logger := s.Logger() image, d := s.hclImage(evalCtx) diags = diags.Extend(d) diff --git a/pkg/handler/handler.go b/pkg/handler/handler.go index bbcdcaa..3105de1 100644 --- a/pkg/handler/handler.go +++ b/pkg/handler/handler.go @@ -172,7 +172,7 @@ func (h *Handler) Update(opts ...HandlerOption) { func (h *Handler) Kill() { signal.Notify(h.Tracker.killSignal, os.Kill) ctx := h.ctx - logger := h.Logger.WithField("watchdog", "") + logger := h.Logger.WithField("orchestra", "watchdog") select { case <-h.Tracker.killSignal: var diags hcl.Diagnostics @@ -203,7 +203,7 @@ func (h *Handler) Kill() { } func (h *Handler) Daemons() { - logger := h.Logger.WithField("watchdog", "") + logger := h.Logger.WithField("orchestra", "watchdog") var completedRunnables ci.Blocks defer h.WriteDiagnostics() @@ -263,7 +263,7 @@ func (h *Handler) Interrupt() { signal.Notify(h.Tracker.interruptSignal, syscall.SIGTERM) ctx := h.ctx - logger := h.Logger.WithField("watchdog", "") + logger := h.Logger.WithField("orchestra", "watchdog") select { case <-h.Tracker.interruptSignal: var diags hcl.Diagnostics diff --git a/pkg/log/logger.go b/pkg/log/logger.go new file mode 100644 index 0000000..7330d54 --- /dev/null +++ b/pkg/log/logger.go @@ -0,0 +1 @@ +package log diff --git a/pkg/orchestra/orchestra.go b/pkg/orchestra/orchestra.go index 4581afd..fd86896 100644 --- a/pkg/orchestra/orchestra.go +++ b/pkg/orchestra/orchestra.go @@ -45,7 +45,7 @@ func Perform(togomak *conductor.Togomak) int { cfg := togomak.Config ctx, cancel := context.WithCancel(togomak.Context) - logger := togomak.Logger + logger := togomak.Logger.WithField("orchestra", "perform") logger.Debugf("starting watchdogs and signal handlers") h := StartHandlers(togomak) @@ -175,7 +175,7 @@ func Perform(togomak *conductor.Togomak) int { h.Tracker.AppendRunnable(runnable) } - go RunWithRetries(runnableId, runnable, ctx, h, logger, opts...) + go RunWithRetries(runnableId, runnable, ctx, h, togomak.Logger, opts...) if cfg.Pipeline.DryRun { // TODO: implement --concurrency option diff --git a/pkg/orchestra/outputs.go b/pkg/orchestra/outputs.go index dd89f0d..684be86 100644 --- a/pkg/orchestra/outputs.go +++ b/pkg/orchestra/outputs.go @@ -15,7 +15,7 @@ import ( func ExpandOutputs(togomak *conductor.Togomak) hcl.Diagnostics { var diags hcl.Diagnostics - logger := togomak.Logger + logger := togomak.Logger.WithField("orchestra", "outputs") togomakEnvFile := filepath.Join(togomak.Process.TempDir, meta.OutputEnvFile) logger.Tracef("%s will be stored and exported here: %s", meta.OutputEnvVar, togomakEnvFile) envFile, err := os.OpenFile(togomakEnvFile, os.O_RDONLY|os.O_CREATE, 0644) diff --git a/pkg/orchestra/run.go b/pkg/orchestra/run.go index c5d6bee..28acfd7 100644 --- a/pkg/orchestra/run.go +++ b/pkg/orchestra/run.go @@ -13,8 +13,9 @@ import ( "time" ) -func RunWithRetries(runnableId string, runnable ci.Block, ctx context.Context, handler *handler.Handler, logger *logrus.Logger, opts ...runnable.Option) { - handler.Logger.Debug("starting runnable with retries ", runnableId) +func RunWithRetries(runnableId string, runnable ci.Block, ctx context.Context, handler *handler.Handler, togomakLogger *logrus.Logger, opts ...runnable.Option) { + logger := togomakLogger.WithField("orchestra", "run") + logger.Debug("starting runnable with retries ", runnableId) stageDiags := runnable.Run(ctx, opts...) handler.Tracker.AppendCompleted(runnable) diff --git a/pkg/ui/colors.go b/pkg/ui/colors.go index 19961e1..ee71ff1 100644 --- a/pkg/ui/colors.go +++ b/pkg/ui/colors.go @@ -16,6 +16,7 @@ var Blue = color.New(color.FgBlue).SprintFunc() var Grey = color.New(color.FgHiBlack).SprintFunc() var Yellow = color.New(color.FgYellow).SprintFunc() var HiYellow = color.New(color.FgHiYellow).SprintFunc() +var HiRed = color.New(color.FgHiRed).SprintFunc() var Italic = color.New(color.Italic).SprintFunc() var Plus = color.New(color.FgHiWhite).SprintFunc()("+") var SubStage = Grey("==>") From ec60ac9023b6cb8c71b6dadf066c7856541dd574 Mon Sep 17 00:00:00 2001 From: Srevin Saju Date: Sat, 14 Oct 2023 22:45:21 +0300 Subject: [PATCH 3/6] refactor: core stagr run logic into a separate function --- pkg/ci/stage_run.go | 62 ++++++++++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/pkg/ci/stage_run.go b/pkg/ci/stage_run.go index 02b59ad..baca410 100644 --- a/pkg/ci/stage_run.go +++ b/pkg/ci/stage_run.go @@ -331,14 +331,9 @@ func (s *Stage) expandMacros(ctx context.Context, opts ...runnable.Option) (*Sta func (s *Stage) Run(ctx context.Context, options ...runnable.Option) (diags hcl.Diagnostics) { logger := s.Logger() - cfg := runnable.NewConfig(options...) - tmpDir := global.TempDir() logger.Debugf("running %s", x.RenderBlock(StageBlock, s.Id)) - status := runnable.StatusRunning - - var err error evalCtx := global.HclEvalContext() // expand stages using macros @@ -347,28 +342,49 @@ func (s *Stage) Run(ctx context.Context, options ...runnable.Option) (diags hcl. diags = diags.Extend(d) logger.Debugf("finished expanding macros with %d errors", len(diags.Errs())) - if s.ForEach != nil { - global.EvalContextMutex.RLock() - forEachItems, d := s.ForEach.Value(evalCtx) - global.EvalContextMutex.RUnlock() - + if s.ForEach == nil { + d = s.run(ctx, evalCtx, options...) diags = diags.Extend(d) - if !forEachItems.IsNull() { - if !forEachItems.CanIterateElements() { - diags = diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "invalid type for for_each", - Detail: fmt.Sprintf("for_each must be a set or map of objects"), - }) - return diags - } + return diags + } - forEachItems.ForEachElement(func(k cty.Value, v cty.Value) bool { - fmt.Println("ok!") - return true + global.EvalContextMutex.RLock() + forEachItems, d := s.ForEach.Value(evalCtx) + global.EvalContextMutex.RUnlock() + + diags = diags.Extend(d) + if d.HasErrors() { + return diags + } + + if !forEachItems.IsNull() { + if !forEachItems.CanIterateElements() { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "invalid type for for_each", + Detail: fmt.Sprintf("for_each must be a set or map of objects"), }) + return diags } + + forEachItems.ForEachElement(func(k cty.Value, v cty.Value) bool { + fmt.Println("ok!", k, v) + return true + }) + } else { + d = s.run(ctx, evalCtx, options...) + diags = diags.Extend(d) + return diags } + return diags +} + +func (s *Stage) run(ctx context.Context, evalCtx *hcl.EvalContext, options ...runnable.Option) (diags hcl.Diagnostics) { + var err error + logger := s.Logger() + tmpDir := global.TempDir() + status := runnable.StatusRunning + cfg := runnable.NewConfig(options...) defer func() { logger.Debug("running post hooks") @@ -388,7 +404,7 @@ func (s *Stage) Run(ctx context.Context, options ...runnable.Option) (diags hcl. logger.Debug("finished running post hooks") }() - d = s.executePreHooks(ctx, status, options...) + d := s.executePreHooks(ctx, status, options...) diags = diags.Extend(d) paramsGo := map[string]cty.Value{} From 46fe2538f98cc1538a92a4f7c75092d939a5777e Mon Sep 17 00:00:00 2001 From: Srevin Saju Date: Sat, 14 Oct 2023 23:30:27 +0300 Subject: [PATCH 4/6] feat: add support for for-each, improve i9n test execution speed --- examples/demo/togomak.hcl | 16 ++--- examples/docker-entrypoint/togomak.hcl | 2 +- examples/git/togomak.hcl | 4 +- examples/hooks/togomak.hcl | 6 +- examples/lifecycles/togomak.hcl | 2 +- examples/pre-post/togomak.hcl | 2 +- examples/terraform/togomak.hcl | 2 +- pkg/ci/runnable.go | 1 + pkg/ci/stage_run.go | 24 ++++++- pkg/runnable/options.go | 12 ++++ tests/togomak.hcl | 87 +++++++++++++++++++------- 11 files changed, 114 insertions(+), 44 deletions(-) diff --git a/examples/demo/togomak.hcl b/examples/demo/togomak.hcl index 038831e..66ab8ab 100644 --- a/examples/demo/togomak.hcl +++ b/examples/demo/togomak.hcl @@ -3,23 +3,23 @@ togomak { } data "prompt" "repo_name" { - prompt = "enter repo name: " + prompt = "enter repo name: " default = "username/repo" } locals { - repo = "srevinsaju/togomak" - lint_tools = ["misspell", "golangci-lint", "abcgo"] + repo = "srevinsaju/togomak" + lint_tools = ["misspell", "golangci-lint", "abcgo"] build_types = ["amd64", "i386", "arm64"] } stage "lint" { script = <<-EOT echo 💅 running style checks for repo ${local.repo} - %{ for tool in local.lint_tools } + %{for tool in local.lint_tools} echo "* running linter: ${tool}" sleep 1 - %{ endfor } + %{endfor} EOT } @@ -27,15 +27,15 @@ stage "lint" { stage "build" { script = <<-EOT echo 👷 running ${ansifmt("green", "build")} - %{ for arch in local.build_types } + %{for arch in local.build_types} echo "* building ${local.repo} for ${arch}..." sleep 1 - %{ endfor } + %{endfor} EOT } stage "deploy" { - if = data.prompt.repo_name.value == "srevinsaju/togomak" + if = data.prompt.repo_name.value == "srevinsaju/togomak" depends_on = [stage.build] container { image = "hashicorp/terraform" diff --git a/examples/docker-entrypoint/togomak.hcl b/examples/docker-entrypoint/togomak.hcl index 5d18508..707142b 100644 --- a/examples/docker-entrypoint/togomak.hcl +++ b/examples/docker-entrypoint/togomak.hcl @@ -4,7 +4,7 @@ togomak { stage "apt" { container { - image = "ubuntu:latest" + image = "ubuntu:latest" entrypoint = ["apt"] } args = ["install"] diff --git a/examples/git/togomak.hcl b/examples/git/togomak.hcl index 746dee4..9476c97 100644 --- a/examples/git/togomak.hcl +++ b/examples/git/togomak.hcl @@ -3,9 +3,9 @@ togomak { } data "git" "repo" { - url = "https://github.com/srevinsaju/togomak" + url = "https://github.com/srevinsaju/togomak" branch = "v1" - files = ["togomak.hcl"] + files = ["togomak.hcl"] } stage "example" { diff --git a/examples/hooks/togomak.hcl b/examples/hooks/togomak.hcl index 70970f5..979d035 100644 --- a/examples/hooks/togomak.hcl +++ b/examples/hooks/togomak.hcl @@ -5,9 +5,9 @@ togomak { stage "example" { script = "echo hello world" - + pre_hook { - stage{ + stage { script = "echo before the script for stage ${this.id} runs" } } @@ -21,7 +21,7 @@ stage "example" { stage "example_2" { script = "echo bye world" - + pre_hook { stage { script = "echo before the script for stage ${this.id} runs" diff --git a/examples/lifecycles/togomak.hcl b/examples/lifecycles/togomak.hcl index ae18a33..d44c0f7 100644 --- a/examples/lifecycles/togomak.hcl +++ b/examples/lifecycles/togomak.hcl @@ -7,7 +7,7 @@ stage "normal" { } stage "dont_execute" { - if = false + if = false script = "echo this shouldnt be executed && exit 1" } diff --git a/examples/pre-post/togomak.hcl b/examples/pre-post/togomak.hcl index d0e3e9e..d58d266 100644 --- a/examples/pre-post/togomak.hcl +++ b/examples/pre-post/togomak.hcl @@ -16,5 +16,5 @@ stage "example" { stage "example_2" { depends_on = [stage.example] - script = "echo hello world 2" + script = "echo hello world 2" } diff --git a/examples/terraform/togomak.hcl b/examples/terraform/togomak.hcl index 948de00..8b2a7d1 100644 --- a/examples/terraform/togomak.hcl +++ b/examples/terraform/togomak.hcl @@ -3,7 +3,7 @@ togomak { } data "tf" "this" { - source = "." + source = "." allow_apply = true } diff --git a/pkg/ci/runnable.go b/pkg/ci/runnable.go index 56fd771..bc4524f 100644 --- a/pkg/ci/runnable.go +++ b/pkg/ci/runnable.go @@ -12,6 +12,7 @@ import ( ) const ThisBlock = "this" +const EachBlock = "each" type Retryable interface { // CanRetry decides if the runnable can be retried diff --git a/pkg/ci/stage_run.go b/pkg/ci/stage_run.go index baca410..8212a69 100644 --- a/pkg/ci/stage_run.go +++ b/pkg/ci/stage_run.go @@ -8,7 +8,9 @@ import ( "github.com/docker/docker/api/types" dockerContainer "github.com/docker/docker/api/types/container" dockerClient "github.com/docker/docker/client" + "github.com/srevinsaju/togomak/v1/pkg/dg" "github.com/srevinsaju/togomak/v1/pkg/x" + "sync" "github.com/docker/docker/pkg/stdcopy" "github.com/google/uuid" @@ -367,16 +369,29 @@ func (s *Stage) Run(ctx context.Context, options ...runnable.Option) (diags hcl. return diags } + var wg sync.WaitGroup + + var safeDg dg.SafeDiagnostics + forEachItems.ForEachElement(func(k cty.Value, v cty.Value) bool { - fmt.Println("ok!", k, v) - return true + id := fmt.Sprintf("%s[\"%s\"]", s.Id, k.AsString()) + wg.Add(1) + stage := &Stage{Id: id, CoreStage: s.CoreStage, Lifecycle: s.Lifecycle} + go func(options ...runnable.Option) { + options = append(options, runnable.WithEach(k, v)) + d := stage.Run(ctx, options...) + safeDg.Extend(d) + wg.Done() + }(options...) + return false }) + wg.Wait() + return safeDg.Diagnostics() } else { d = s.run(ctx, evalCtx, options...) diags = diags.Extend(d) return diags } - return diags } func (s *Stage) run(ctx context.Context, evalCtx *hcl.EvalContext, options ...runnable.Option) (diags hcl.Diagnostics) { @@ -438,6 +453,9 @@ func (s *Stage) run(ctx context.Context, evalCtx *hcl.EvalContext, options ...ru "status": cty.StringVal(string(cfg.Status.Status)), }), } + if cfg.Each != nil { + evalCtx.Variables[EachBlock] = cty.ObjectVal(cfg.Each) + } logger.Debugf("expanding macro parameters") if s.Use != nil && s.Use.Parameters != nil { diff --git a/pkg/runnable/options.go b/pkg/runnable/options.go index 2288623..1b5e74f 100644 --- a/pkg/runnable/options.go +++ b/pkg/runnable/options.go @@ -3,6 +3,7 @@ package runnable import ( "github.com/srevinsaju/togomak/v1/pkg/behavior" "github.com/srevinsaju/togomak/v1/pkg/path" + "github.com/zclconf/go-cty/cty" ) type Config struct { @@ -12,6 +13,8 @@ type Config struct { Paths *path.Path + Each map[string]cty.Value + Behavior *behavior.Behavior } @@ -42,6 +45,15 @@ func WithParent(parent ParentConfig) Option { } } +func WithEach(k cty.Value, v cty.Value) Option { + return func(c *Config) { + c.Each = map[string]cty.Value{ + "key": k, + "value": v, + } + } +} + func WithBehavior(behavior *behavior.Behavior) Option { return func(c *Config) { c.Behavior = behavior diff --git a/tests/togomak.hcl b/tests/togomak.hcl index 0fd43cb..18f806c 100644 --- a/tests/togomak.hcl +++ b/tests/togomak.hcl @@ -24,34 +24,73 @@ stage "coverage_prepare" { EOT } -stage "integration_tests" { +stage "tests" { + pre_hook { + stage { + script = "echo ${ansi.fg.green}${each.key}${ansi.reset}: full" + } + } + depends_on = [stage.build, stage.coverage_prepare] + for_each = fileset(cwd, "../examples/*/togomak.hcl") + args = [ + "./togomak_coverage", + "-C", dirname(each.key), + "--ci", "-v", "-v", "-v", + ] + + env { + name = "GOCOVERDIR" + value = local.coverage_data_dir + } +} + + +stage "tests_dry_run" { + pre_hook { + stage { + script = "echo ${ansi.fg.green}${each.key}${ansi.reset}: dry" + } + } + + depends_on = [stage.build, stage.coverage_prepare] + for_each = fileset(cwd, "../examples/*/togomak.hcl") + args = [ + "./togomak_coverage", + "-C", dirname(each.key), + "--ci", "-v", "-v", "-v", "-n", + ] + + env { + name = "GOCOVERDIR" + value = local.coverage_data_dir + } +} + +stage "fmt" { + depends_on = [stage.build, stage.coverage_prepare] + script = "./togomak_coverage fmt --check --recursive" +} + +stage "cache" { + depends_on = [stage.fmt, stage.tests, stage.tests_dry_run] + script = "./togomak_coverage cache clean --recursive" +} + + +stage "failing" { + depends_on = [stage.cache] + for_each = fileset(cwd, "tests/failing/*/togomak.hcl") script = <<-EOT - #!/usr/bin/env bash - set -e - ls ../examples - for i in ../examples/*; do - echo ${ansi.fg.green}$i${ansi.reset} - ./togomak_coverage -C "$i" --ci -v -v -v - ./togomak_coverage -C "$i" --ci -v -v -v root - ./togomak_coverage -C "$i" --ci -v -v -v -n - done - ./togomak_coverage cache clean --recursive - ./togomak_coverage fmt --check --recursive - - for i in tests/failing/*; do - set +e - echo ${ansi.fg.green}$i${ansi.reset} - ./togomak_coverage -C "$i" --ci -v -v -v - result=$? - if [ $result -eq 0 ]; then + set +e + ./togomak_coverage -C "${dirname(each.key)}" --ci -v -v -v + result=$? + if [ $result -eq 0 ]; then set -e echo "$i completed successfully when it was supposed to fail" exit 1 - fi - done - EOT - + fi + EOT env { name = "GOCOVERDIR" value = local.coverage_data_dir @@ -59,7 +98,7 @@ stage "integration_tests" { } stage "coverage_raw" { - depends_on = [stage.integration_tests] + depends_on = [stage.tests] script = "go tool covdata percent -i=${local.coverage_data_dir}" } stage "coverage_merge" { From ea4ee420dc5a7372ce6736d124a5705730330fde Mon Sep 17 00:00:00 2001 From: Srevin Saju Date: Sat, 14 Oct 2023 23:38:01 +0300 Subject: [PATCH 5/6] feat: init changelog --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..c72a40c --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,9 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] +- Add support for `for_each` meta argument From bc9a36b0a10b99dc9e9675d5690e73379995f3c0 Mon Sep 17 00:00:00 2001 From: Srevin Saju Date: Sat, 14 Oct 2023 23:38:50 +0300 Subject: [PATCH 6/6] test(for_each): add for each map test --- examples/for-each-map/togomak.hcl | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 examples/for-each-map/togomak.hcl diff --git a/examples/for-each-map/togomak.hcl b/examples/for-each-map/togomak.hcl new file mode 100644 index 0000000..5830451 --- /dev/null +++ b/examples/for-each-map/togomak.hcl @@ -0,0 +1,21 @@ +togomak { + version = 2 +} + +locals { + m = { + part1 = "You Are (Not) Alone." + part2 = "You Can (Not) Advance." + part3 = "You Can (Not) Redo." + part3-1 = "Thrice Upon a Time." + } +} + + +stage "movie" { + for_each = local.m + name = "example" + script = <<-EOT + echo "Evangelion ${each.key}: ${each.value}" + EOT +}