diff --git a/docs/src/configuration/macro.md b/docs/src/configuration/macro.md index cc238cb..8e5939c 100644 --- a/docs/src/configuration/macro.md +++ b/docs/src/configuration/macro.md @@ -37,7 +37,7 @@ For example, files = { "togomak.hcl" = <<-EOT togomak { - version = 1 + version = 2 } stage "hello" { script = "echo hello world" diff --git a/docs/src/configuration/stage.md b/docs/src/configuration/stage.md index f6ac818..ae7cbbc 100644 --- a/docs/src/configuration/stage.md +++ b/docs/src/configuration/stage.md @@ -10,7 +10,7 @@ of arguments, or a macro. ### Stage with Script ```hcl ~togomak { -~ version = 1 +~ version = 2 ~} ~ stage "script" { @@ -21,7 +21,7 @@ stage "script" { ### Stage with Command and Arguments ```hcl ~togomak { -~ version = 1 +~ version = 2 ~} ~ stage "command" { @@ -32,7 +32,7 @@ stage "command" { ### Stage with Macro ```hcl ~togomak { -~ version = 1 +~ version = 2 ~} ~ macro "echo" { @@ -51,7 +51,7 @@ stage "macro" { ### Stage with Dependencies ```hcl ~togomak { -~ version = 1 +~ version = 2 ~} ~ stage "build" { @@ -67,7 +67,7 @@ stage "install" { ### Stage with Retry ```hcl ~togomak { -~ version = 1 +~ version = 2 ~} ~ stage "build" { diff --git a/docs/src/configuration/togomak.md b/docs/src/configuration/togomak.md index 3487f06..99a8368 100644 --- a/docs/src/configuration/togomak.md +++ b/docs/src/configuration/togomak.md @@ -9,7 +9,7 @@ block for the file to be recognized as a valid togomak { # ... - version = 1 + version = 2 } ``` The above block, with `version` parameter diff --git a/examples/conditions/togomak.hcl b/examples/conditions/togomak.hcl index 9c3af6e..204f200 100644 --- a/examples/conditions/togomak.hcl +++ b/examples/conditions/togomak.hcl @@ -1,5 +1,5 @@ togomak { - version = 1 + version = 2 } data "env" "home" { diff --git a/examples/docker/togomak.hcl b/examples/docker/togomak.hcl index 5c0dbc9..8b9ce35 100644 --- a/examples/docker/togomak.hcl +++ b/examples/docker/togomak.hcl @@ -1,5 +1,5 @@ togomak { - version = 1 + version = 2 } stage "example" { diff --git a/examples/env/togomak.hcl b/examples/env/togomak.hcl index 65a586a..ca7feaf 100644 --- a/examples/env/togomak.hcl +++ b/examples/env/togomak.hcl @@ -1,5 +1,5 @@ togomak { - version = 1 + version = 2 } data "env" "HOME" { diff --git a/examples/files/togomak.hcl b/examples/files/togomak.hcl index 55ef18d..45a8e41 100644 --- a/examples/files/togomak.hcl +++ b/examples/files/togomak.hcl @@ -1,5 +1,5 @@ togomak { - version = 1 + version = 2 } diff --git a/examples/git/togomak.hcl b/examples/git/togomak.hcl index 3d8bbd2..36f3caa 100644 --- a/examples/git/togomak.hcl +++ b/examples/git/togomak.hcl @@ -1,5 +1,5 @@ togomak { - version = 1 + version = 2 } data "git" "repo" { diff --git a/examples/git_tags/togomak.hcl b/examples/git_tags/togomak.hcl index 7572fca..8eb215b 100644 --- a/examples/git_tags/togomak.hcl +++ b/examples/git_tags/togomak.hcl @@ -1,5 +1,5 @@ togomak { - version = 1 + version = 2 } data "git" "repo" { diff --git a/examples/locals/togomak.hcl b/examples/locals/togomak.hcl index a0d5576..998ff38 100644 --- a/examples/locals/togomak.hcl +++ b/examples/locals/togomak.hcl @@ -1,5 +1,5 @@ togomak { - version = 1 + version = 2 } locals { diff --git a/examples/macros/togomak.hcl b/examples/macros/togomak.hcl index 61e24fc..10ba065 100644 --- a/examples/macros/togomak.hcl +++ b/examples/macros/togomak.hcl @@ -1,5 +1,5 @@ togomak { - version = 1 + version = 2 } macro "explode" { diff --git a/examples/multiple-files/stage1.hcl b/examples/multiple-files/stage1.hcl new file mode 100644 index 0000000..4dc751e --- /dev/null +++ b/examples/multiple-files/stage1.hcl @@ -0,0 +1,4 @@ +stage "example" { + name = "example" + script = "echo hello world" +} diff --git a/examples/multiple-files/stage2.hcl b/examples/multiple-files/stage2.hcl new file mode 100644 index 0000000..cca32a6 --- /dev/null +++ b/examples/multiple-files/stage2.hcl @@ -0,0 +1,3 @@ +stage "example_2" { + script = "echo bye world" +} diff --git a/examples/multiple-files/togomak.hcl b/examples/multiple-files/togomak.hcl new file mode 100644 index 0000000..75ffef8 --- /dev/null +++ b/examples/multiple-files/togomak.hcl @@ -0,0 +1,4 @@ +togomak { + version = 2 +} + diff --git a/examples/output/togomak.hcl b/examples/output/togomak.hcl index 0a98b11..7108e7f 100644 --- a/examples/output/togomak.hcl +++ b/examples/output/togomak.hcl @@ -1,5 +1,5 @@ togomak { - version = 1 + version = 2 } stage "agent" { diff --git a/examples/prompt/togomak.hcl b/examples/prompt/togomak.hcl index 1511586..f5e60c3 100644 --- a/examples/prompt/togomak.hcl +++ b/examples/prompt/togomak.hcl @@ -1,5 +1,5 @@ togomak { - version = 1 + version = 2 } data "env" "quit_if_not_shinji" { diff --git a/examples/remote-stages/togomak.hcl b/examples/remote-stages/togomak.hcl index 4b92cd8..322a29e 100644 --- a/examples/remote-stages/togomak.hcl +++ b/examples/remote-stages/togomak.hcl @@ -1,5 +1,5 @@ togomak { - version = 1 + version = 2 } data "git" "eva01_source" { diff --git a/pkg/ci/builder.go b/pkg/ci/builder.go index 0783036..03938e5 100644 --- a/pkg/ci/builder.go +++ b/pkg/ci/builder.go @@ -3,5 +3,5 @@ package ci const BuilderBlock = "togomak" type Builder struct { - Version string `hcl:"version" json:"version"` + Version int `hcl:"version" json:"version"` } diff --git a/pkg/ci/data_prop.go b/pkg/ci/data_prop.go new file mode 100644 index 0000000..214f10e --- /dev/null +++ b/pkg/ci/data_prop.go @@ -0,0 +1,5 @@ +package ci + +func (d *Data) Override() bool { + return true +} diff --git a/pkg/ci/locals_prop.go b/pkg/ci/locals_prop.go new file mode 100644 index 0000000..35434ed --- /dev/null +++ b/pkg/ci/locals_prop.go @@ -0,0 +1,5 @@ +package ci + +func (l *Local) Override() bool { + return false +} diff --git a/pkg/ci/macro_prop.go b/pkg/ci/macro_prop.go new file mode 100644 index 0000000..003e82a --- /dev/null +++ b/pkg/ci/macro_prop.go @@ -0,0 +1,5 @@ +package ci + +func (m *Macro) Override() bool { + return true +} diff --git a/pkg/ci/prop.go b/pkg/ci/prop.go new file mode 100644 index 0000000..3efdf7b --- /dev/null +++ b/pkg/ci/prop.go @@ -0,0 +1,10 @@ +package ci + +type Overrideable interface { + Override() bool +} + +type Distinct interface { + Overrideable + Describable +} diff --git a/pkg/ci/stage_prop.go b/pkg/ci/stage_prop.go new file mode 100644 index 0000000..af25dfe --- /dev/null +++ b/pkg/ci/stage_prop.go @@ -0,0 +1,9 @@ +package ci + +func (s Stage) Override() bool { + return false +} + +func (s Stages) Override() bool { + return false +} diff --git a/pkg/ci/stage_schema.go b/pkg/ci/stage_schema.go index 3bcfa79..4c405f7 100644 --- a/pkg/ci/stage_schema.go +++ b/pkg/ci/stage_schema.go @@ -33,6 +33,24 @@ type StageContainer struct { type Stages []Stage +// IsDistinct checks if the stages in s and ss are distinct +// TODO: check if this is a good way to do this +func (s Stages) CheckIfDistinct(ss Stages) hcl.Diagnostics { + var diags hcl.Diagnostics + for _, stage := range s { + for _, stage2 := range ss { + if stage.Id == stage2.Id { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Duplicate stage", + Detail: "Stage with id " + stage.Id + " is defined more than once", + }) + } + } + } + return diags +} + type StageEnvironment struct { Name string `hcl:"name" json:"name"` Value hcl.Expression `hcl:"value" json:"value"` diff --git a/pkg/orchestra/format.go b/pkg/orchestra/format.go index cf7c413..8e5944a 100644 --- a/pkg/orchestra/format.go +++ b/pkg/orchestra/format.go @@ -7,6 +7,7 @@ import ( "github.com/hashicorp/hcl/v2/hclwrite" "github.com/srevinsaju/togomak/v1/pkg/pipeline" "os" + "path/filepath" ) func Format(cfg Config, check bool, recursive bool) error { @@ -32,15 +33,23 @@ func Format(cfg Config, check bool, recursive bool) error { t.Logger.Fatalf("Error while globbing for **/*.hcl: %s", err) } } else { - fn := pipeline.ConfigFilePath(ctx) - data, err := os.ReadFile(fn) + fDir := pipeline.ConfigFileDir(ctx) + fNames, err := os.ReadDir(fDir) if err != nil { - return err + panic(err) } - outSrc := hclwrite.Format(data) - if !bytes.Equal(outSrc, data) { - t.Logger.Tracef("%s needs formatting", fn) - toFormat = append(toFormat, fn) + + for _, f := range fNames { + fn := filepath.Join(fDir, f.Name()) + data, err := os.ReadFile(fn) + if err != nil { + return err + } + outSrc := hclwrite.Format(data) + if !bytes.Equal(outSrc, data) { + t.Logger.Tracef("%s needs formatting", fn) + toFormat = append(toFormat, fn) + } } } for _, fn := range toFormat { diff --git a/pkg/pipeline/parse.go b/pkg/pipeline/parse.go index 5291942..019ae4d 100644 --- a/pkg/pipeline/parse.go +++ b/pkg/pipeline/parse.go @@ -2,15 +2,22 @@ package pipeline import ( "context" + "fmt" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/gohcl" "github.com/hashicorp/hcl/v2/hclparse" "github.com/srevinsaju/togomak/v1/pkg/c" "github.com/srevinsaju/togomak/v1/pkg/ci" "github.com/srevinsaju/togomak/v1/pkg/meta" + "github.com/srevinsaju/togomak/v1/pkg/ui" + "os" "path/filepath" + "strings" ) +// configFilePath returns the path to the configuration file. If the path is not absolute, it is assumed to be +// relative to the working directory +// DEPRECATED: use configFileDir instead func ConfigFilePath(ctx context.Context) string { filePath := ctx.Value(c.TogomakContextPipelineFilePath).(string) if filePath == "" { @@ -24,16 +31,109 @@ func ConfigFilePath(ctx context.Context) string { return filePath } -func Read(ctx context.Context, parser *hclparse.Parser) (*ci.Pipeline, hcl.Diagnostics) { - filePath := ConfigFilePath(ctx) +func ConfigFileDir(ctx context.Context) string { + return filepath.Dir(ConfigFilePath(ctx)) +} - f, diags := parser.ParseHCLFile(filePath) +// Read reads togomak.hcl from the configuration file directory. A configuration file directory is the one that +// contains togomak.hcl, it searches recursively outwards. +// DEPRECATED: use ReadDir instead +func Read(ctx context.Context, parser *hclparse.Parser) (*ci.Pipeline, hcl.Diagnostics) { + ciFile := ConfigFilePath(ctx) + f, diags := parser.ParseHCLFile(ciFile) if diags.HasErrors() { return nil, diags } pipeline := &ci.Pipeline{} diags = gohcl.DecodeBody(f.Body, nil, pipeline) + + if pipeline.Builder.Version != 1 { + return ReadDir(ctx, parser) + } else if pipeline.Builder.Version == 1 { + ui.DeprecationWarning(fmt.Sprintf("%s configuration version 1 is deprecated, and support for the same will be removed in a later version. ", meta.AppName)) + } return pipeline, diags } + +// ReadDir parses an entire directory of *.hcl files and merges them together. This is useful when you want to +// split your pipeline into multiple files, without having to import them individually +func ReadDir(ctx context.Context, parser *hclparse.Parser) (*ci.Pipeline, hcl.Diagnostics) { + var diags hcl.Diagnostics + dir := ConfigFileDir(ctx) + togomakFiles, err := os.ReadDir(dir) + if err != nil { + return nil, diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "directory not found", + Detail: err.Error(), + }) + } + var pipes []*pipelineMeta + for _, file := range togomakFiles { + if file.IsDir() { + continue + } + if !strings.HasSuffix(file.Name(), ".hcl") { + continue + } + f, d := parser.ParseHCLFile(filepath.Join(dir, file.Name())) + diags = diags.Extend(d) + + p := &ci.Pipeline{} + + d = gohcl.DecodeBody(f.Body, nil, p) + diags = diags.Extend(d) + pipes = append(pipes, &pipelineMeta{ + pipe: p, + f: f, + filename: file.Name(), + }) + + } + return createRawPipeline(pipes...) + +} + +// pipelineMeta is a helper struct to create a pipeline from multiple pipelines +// this additionally includes the file pointer f, and the filename +type pipelineMeta struct { + pipe *ci.Pipeline + f *hcl.File + filename string +} + +// createRawPipeline creates a pipeline from multiple pipelines. This is useful when you want to merge multiple +// pipelines together, without having to import them individually +func createRawPipeline(pipelines ...*pipelineMeta) (*ci.Pipeline, hcl.Diagnostics) { + pipe := &ci.Pipeline{} + + var diags hcl.Diagnostics + + var versionDefinedFromFilename string + for _, p := range pipelines { + if pipe.Builder.Version == 0 && p.pipe.Builder.Version != 0 { + pipe.Builder.Version = p.pipe.Builder.Version + versionDefinedFromFilename = p.filename + } + if p.pipe.Builder.Version != pipe.Builder.Version && p.pipe.Builder.Version != 0 { + // when overriding and using multiple pipelines, the version of the togomak pipeline schema is + // required to be the same + return nil, diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "version mismatch", + Detail: fmt.Sprintf("version mismatch between pipelines: %d (%s) and %d (%s)", p.pipe.Builder.Version, p.filename, pipe.Builder.Version, versionDefinedFromFilename), + }) + } + + // TODO: create an error if there are duplicate resource definition + pipe.Stages = append(pipe.Stages, p.pipe.Stages...) + pipe.Data = append(pipe.Data, p.pipe.Data...) + pipe.DataProviders = append(pipe.DataProviders, p.pipe.DataProviders...) + pipe.Macros = append(pipe.Macros, p.pipe.Macros...) + pipe.Locals = append(pipe.Locals, p.pipe.Locals...) + pipe.Local = append(pipe.Local, p.pipe.Local...) + } + return pipe, diags +} diff --git a/pkg/ui/colors.go b/pkg/ui/colors.go index cd26658..3f9415b 100644 --- a/pkg/ui/colors.go +++ b/pkg/ui/colors.go @@ -13,6 +13,7 @@ var Bold = color.New(color.Bold).SprintFunc() var Blue = color.New(color.FgBlue).SprintFunc() var Grey = color.New(color.FgHiBlack).SprintFunc() var Yellow = color.New(color.FgYellow).SprintFunc() +var HiYellow = color.New(color.FgHiYellow).SprintFunc() var Italic = color.New(color.Italic).SprintFunc() var Plus = color.New(color.FgHiWhite).SprintFunc()("+") var SubStage = Grey("==>") @@ -33,3 +34,7 @@ func Error(message string) { func Success(message string, args ...interface{}) { fmt.Println(Green(fmt.Sprintf(message, args...))) } + +func DeprecationWarning(message string, args ...string) { + fmt.Println(HiYellow("[deprecated] "), message, args) +} diff --git a/tests/tests/failing/cant-run-fail/togomak.hcl b/tests/tests/failing/cant-run-fail/togomak.hcl index d987877..9d8030f 100644 --- a/tests/tests/failing/cant-run-fail/togomak.hcl +++ b/tests/tests/failing/cant-run-fail/togomak.hcl @@ -1,5 +1,5 @@ togomak { - version = 1 + version = 2 } stage "example" { if = this.what diff --git a/tests/tests/failing/dependency-cycles/togomak.hcl b/tests/tests/failing/dependency-cycles/togomak.hcl index bab301d..1058474 100644 --- a/tests/tests/failing/dependency-cycles/togomak.hcl +++ b/tests/tests/failing/dependency-cycles/togomak.hcl @@ -1,5 +1,5 @@ togomak { - version = 1 + version = 2 } stage "example_a" { diff --git a/tests/tests/failing/env-data-key-missing/togomak.hcl b/tests/tests/failing/env-data-key-missing/togomak.hcl index e632d6d..5bc5059 100644 --- a/tests/tests/failing/env-data-key-missing/togomak.hcl +++ b/tests/tests/failing/env-data-key-missing/togomak.hcl @@ -1,5 +1,5 @@ togomak { - version = 1 + version = 2 } data "env" "hello" { diff --git a/tests/tests/failing/invalid-block-id/togomak.hcl b/tests/tests/failing/invalid-block-id/togomak.hcl index 4a1fa5e..8f7340a 100644 --- a/tests/tests/failing/invalid-block-id/togomak.hcl +++ b/tests/tests/failing/invalid-block-id/togomak.hcl @@ -1,5 +1,5 @@ togomak { - version = 1 + version = 2 } data "env" "hello_world" { diff --git a/tests/tests/failing/invalid-env-output/togomak.hcl b/tests/tests/failing/invalid-env-output/togomak.hcl index 139e848..14fa226 100644 --- a/tests/tests/failing/invalid-env-output/togomak.hcl +++ b/tests/tests/failing/invalid-env-output/togomak.hcl @@ -1,5 +1,5 @@ togomak { - version = 1 + version = 2 } stage "agent" { diff --git a/tests/tests/failing/invalid-stage-macro-invalid-file/togomak.hcl b/tests/tests/failing/invalid-stage-macro-invalid-file/togomak.hcl index 6860dbb..6996d47 100644 --- a/tests/tests/failing/invalid-stage-macro-invalid-file/togomak.hcl +++ b/tests/tests/failing/invalid-stage-macro-invalid-file/togomak.hcl @@ -1,5 +1,5 @@ togomak { - version = 1 + version = 2 } macro "pen_pen" { source = "pen_pen.hcl" diff --git a/tests/tests/failing/invalid-stage-macro/togomak.hcl b/tests/tests/failing/invalid-stage-macro/togomak.hcl index f537269..ed1513c 100644 --- a/tests/tests/failing/invalid-stage-macro/togomak.hcl +++ b/tests/tests/failing/invalid-stage-macro/togomak.hcl @@ -1,5 +1,5 @@ togomak { - version = 1 + version = 2 } locals { diff --git a/tests/tests/failing/invalid-stage-multiple-macros/togomak.hcl b/tests/tests/failing/invalid-stage-multiple-macros/togomak.hcl index 842e6c3..c8e3392 100644 --- a/tests/tests/failing/invalid-stage-multiple-macros/togomak.hcl +++ b/tests/tests/failing/invalid-stage-multiple-macros/togomak.hcl @@ -1,5 +1,5 @@ togomak { - version = 1 + version = 2 } macro "rei" { stage "this" { diff --git a/tests/tests/failing/nested-failing-tests/nested.hcl b/tests/tests/failing/nested-failing-tests/nested.hcl index 75df13e..a80222d 100644 --- a/tests/tests/failing/nested-failing-tests/nested.hcl +++ b/tests/tests/failing/nested-failing-tests/nested.hcl @@ -1,5 +1,5 @@ togomak { - version = 1 + version = 2 } stage "test" { diff --git a/tests/tests/failing/nested-failing-tests/togomak.hcl b/tests/tests/failing/nested-failing-tests/togomak.hcl index 0cfa0ae..8ac69d0 100644 --- a/tests/tests/failing/nested-failing-tests/togomak.hcl +++ b/tests/tests/failing/nested-failing-tests/togomak.hcl @@ -1,5 +1,5 @@ togomak { - version = 1 + version = 2 } macro "nested" { diff --git a/tests/tests/failing/retry/togomak.hcl b/tests/tests/failing/retry/togomak.hcl index 5752e41..803fd95 100644 --- a/tests/tests/failing/retry/togomak.hcl +++ b/tests/tests/failing/retry/togomak.hcl @@ -1,5 +1,5 @@ togomak { - version = 1 + version = 2 } diff --git a/tests/tests/failing/self-referenced-dependencies/togomak.hcl b/tests/tests/failing/self-referenced-dependencies/togomak.hcl index d41d257..aa7ba0a 100644 --- a/tests/tests/failing/self-referenced-dependencies/togomak.hcl +++ b/tests/tests/failing/self-referenced-dependencies/togomak.hcl @@ -1,5 +1,5 @@ togomak { - version = 1 + version = 2 } stage "example" { depends_on = [stage.example] diff --git a/tests/togomak.hcl b/tests/togomak.hcl index 75edef0..f306be2 100644 --- a/tests/togomak.hcl +++ b/tests/togomak.hcl @@ -1,5 +1,5 @@ togomak { - version = 1 + version = 2 } diff --git a/togomak.hcl b/togomak.hcl index 545eff3..02cc1d1 100644 --- a/togomak.hcl +++ b/togomak.hcl @@ -1,5 +1,5 @@ togomak { - version = 1 + version = 2 }