feat: add support for loading multiple files on the same directory

docs/src/configuration/macro.md

@@ -37,7 +37,7 @@ For example,
 files = {
     "togomak.hcl" = <<-EOT
     togomak {
-        version = 1
+        version = 2
     }
     stage "hello" {
         script = "echo hello world"

docs/src/configuration/stage.md

@@ -10,7 +10,7 @@ of arguments, or a macro.
 ### Stage with Script 
 ```hcl 
 ~togomak {
-~  version = 1
+~  version = 2
 ~}
 ~
 stage "script" {
@@ -21,7 +21,7 @@ stage "script" {
 ### Stage with Command and Arguments
 ```hcl
 ~togomak {
-~  version = 1
+~  version = 2
 ~}
 ~
 stage "command" {
@@ -32,7 +32,7 @@ stage "command" {
 ### Stage with Macro
 ```hcl
 ~togomak {
-~  version = 1
+~  version = 2
 ~}
 ~
 macro "echo" {
@@ -51,7 +51,7 @@ stage "macro" {
 ### Stage with Dependencies
 ```hcl
 ~togomak {
-~  version = 1
+~  version = 2
 ~}
 ~
 stage "build" {
@@ -67,7 +67,7 @@ stage "install" {
 ### Stage with Retry
 ```hcl
 ~togomak {
-~  version = 1
+~  version = 2
 ~}
 ~
 stage "build" {

docs/src/configuration/togomak.md

@@ -9,7 +9,7 @@ block for the file to be recognized as a valid
 
 togomak {
     # ...
-    version = 1
+    version = 2
 }
 ```
 The above block, with `version` parameter 

examples/conditions/togomak.hcl

@@ -1,5 +1,5 @@
 togomak {
-  version = 1
+  version = 2
 }
 
 data "env" "home" {

examples/docker/togomak.hcl

@@ -1,5 +1,5 @@
 togomak {
-  version = 1
+  version = 2
 }
 
 stage "example" {

examples/env/togomak.hcl

@@ -1,5 +1,5 @@
 togomak {
-  version = 1
+  version = 2
 }
 
 data "env" "HOME" {

examples/files/togomak.hcl

@@ -1,5 +1,5 @@
 togomak {
-  version = 1
+  version = 2
 }
 
 

examples/git/togomak.hcl

@@ -1,5 +1,5 @@
 togomak {
-  version = 1
+  version = 2
 }
 
 data "git" "repo" {

examples/git_tags/togomak.hcl

@@ -1,5 +1,5 @@
 togomak {
-  version = 1
+  version = 2
 }
 
 data "git" "repo" {

examples/locals/togomak.hcl

@@ -1,5 +1,5 @@
 togomak {
-  version = 1
+  version = 2
 }
 
 locals {

examples/macros/togomak.hcl

@@ -1,5 +1,5 @@
 togomak {
-  version = 1
+  version = 2
 }
 
 macro "explode" {

examples/multiple-files/stage1.hcl

@@ -0,0 +1,4 @@
+stage "example" {
+  name   = "example"
+  script = "echo hello world"
+}

examples/multiple-files/stage2.hcl

@@ -0,0 +1,3 @@
+stage "example_2" {
+  script = "echo bye world"
+}

examples/multiple-files/togomak.hcl

@@ -0,0 +1,4 @@
+togomak {
+  version = 2
+}
+

examples/output/togomak.hcl

@@ -1,5 +1,5 @@
 togomak {
-  version = 1
+  version = 2
 }
 
 stage "agent" {

examples/prompt/togomak.hcl

@@ -1,5 +1,5 @@
 togomak {
-  version = 1
+  version = 2
 }
 
 data "env" "quit_if_not_shinji" {

examples/remote-stages/togomak.hcl

@@ -1,5 +1,5 @@
 togomak {
-  version = 1
+  version = 2
 }
 
 data "git" "eva01_source" {

pkg/ci/builder.go

@@ -3,5 +3,5 @@ package ci
 const BuilderBlock = "togomak"
 
 type Builder struct {
-	Version string `hcl:"version" json:"version"`
+	Version int `hcl:"version" json:"version"`
 }

pkg/ci/data_prop.go

@@ -0,0 +1,5 @@
+package ci
+
+func (d *Data) Override() bool {
+	return true
+}

pkg/ci/locals_prop.go

@@ -0,0 +1,5 @@
+package ci
+
+func (l *Local) Override() bool {
+	return false
+}

pkg/ci/macro_prop.go

@@ -0,0 +1,5 @@
+package ci
+
+func (m *Macro) Override() bool {
+	return true
+}

pkg/ci/prop.go

@@ -0,0 +1,10 @@
+package ci
+
+type Overrideable interface {
+	Override() bool
+}
+
+type Distinct interface {
+	Overrideable
+	Describable
+}

pkg/ci/stage_prop.go

@@ -0,0 +1,9 @@
+package ci
+
+func (s Stage) Override() bool {
+	return false
+}
+
+func (s Stages) Override() bool {
+	return false
+}

pkg/ci/stage_schema.go

@@ -33,6 +33,24 @@ type StageContainer struct {
 
 type Stages []Stage
 
+// IsDistinct checks if the stages in s and ss are distinct
+// TODO: check if this is a good way to do this
+func (s Stages) CheckIfDistinct(ss Stages) hcl.Diagnostics {
+	var diags hcl.Diagnostics
+	for _, stage := range s {
+		for _, stage2 := range ss {
+			if stage.Id == stage2.Id {
+				diags = append(diags, &hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "Duplicate stage",
+					Detail:   "Stage with id " + stage.Id + " is defined more than once",
+				})
+			}
+		}
+	}
+	return diags
+}
+
 type StageEnvironment struct {
 	Name  string         `hcl:"name" json:"name"`
 	Value hcl.Expression `hcl:"value" json:"value"`

pkg/orchestra/format.go

@@ -7,6 +7,7 @@ import (
 	"github.com/hashicorp/hcl/v2/hclwrite"
 	"github.com/srevinsaju/togomak/v1/pkg/pipeline"
 	"os"
+	"path/filepath"
 )
 
 func Format(cfg Config, check bool, recursive bool) error {
@@ -32,15 +33,23 @@ func Format(cfg Config, check bool, recursive bool) error {
 			t.Logger.Fatalf("Error while globbing for **/*.hcl: %s", err)
 		}
 	} else {
-		fn := pipeline.ConfigFilePath(ctx)
-		data, err := os.ReadFile(fn)
+		fDir := pipeline.ConfigFileDir(ctx)
+		fNames, err := os.ReadDir(fDir)
 		if err != nil {
-			return err
+			panic(err)
 		}
-		outSrc := hclwrite.Format(data)
-		if !bytes.Equal(outSrc, data) {
-			t.Logger.Tracef("%s needs formatting", fn)
-			toFormat = append(toFormat, fn)
+
+		for _, f := range fNames {
+			fn := filepath.Join(fDir, f.Name())
+			data, err := os.ReadFile(fn)
+			if err != nil {
+				return err
+			}
+			outSrc := hclwrite.Format(data)
+			if !bytes.Equal(outSrc, data) {
+				t.Logger.Tracef("%s needs formatting", fn)
+				toFormat = append(toFormat, fn)
+			}
 		}
 	}
 	for _, fn := range toFormat {

pkg/pipeline/parse.go

@@ -2,15 +2,22 @@ package pipeline
 
 import (
 	"context"
+	"fmt"
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/hcl/v2/gohcl"
 	"github.com/hashicorp/hcl/v2/hclparse"
 	"github.com/srevinsaju/togomak/v1/pkg/c"
 	"github.com/srevinsaju/togomak/v1/pkg/ci"
 	"github.com/srevinsaju/togomak/v1/pkg/meta"
+	"github.com/srevinsaju/togomak/v1/pkg/ui"
+	"os"
 	"path/filepath"
+	"strings"
 )
 
+// configFilePath returns the path to the configuration file. If the path is not absolute, it is assumed to be
+// relative to the working directory
+// DEPRECATED: use configFileDir instead
 func ConfigFilePath(ctx context.Context) string {
 	filePath := ctx.Value(c.TogomakContextPipelineFilePath).(string)
 	if filePath == "" {
@@ -24,16 +31,109 @@ func ConfigFilePath(ctx context.Context) string {
 	return filePath
 }
 
-func Read(ctx context.Context, parser *hclparse.Parser) (*ci.Pipeline, hcl.Diagnostics) {
-	filePath := ConfigFilePath(ctx)
+func ConfigFileDir(ctx context.Context) string {
+	return filepath.Dir(ConfigFilePath(ctx))
+}
 
-	f, diags := parser.ParseHCLFile(filePath)
+// Read reads togomak.hcl from the configuration file directory. A configuration file directory is the one that
+// contains togomak.hcl, it searches recursively outwards.
+// DEPRECATED: use ReadDir instead
+func Read(ctx context.Context, parser *hclparse.Parser) (*ci.Pipeline, hcl.Diagnostics) {
+	ciFile := ConfigFilePath(ctx)
 
+	f, diags := parser.ParseHCLFile(ciFile)
 	if diags.HasErrors() {
 		return nil, diags
 	}
 
 	pipeline := &ci.Pipeline{}
 	diags = gohcl.DecodeBody(f.Body, nil, pipeline)
+
+	if pipeline.Builder.Version != 1 {
+		return ReadDir(ctx, parser)
+	} else if pipeline.Builder.Version == 1 {
+		ui.DeprecationWarning(fmt.Sprintf("%s configuration version 1 is deprecated, and support for the same will be removed in a later version. ", meta.AppName))
+	}
 	return pipeline, diags
 }
+
+// ReadDir parses an entire directory of *.hcl files and merges them together. This is useful when you want to
+// split your pipeline into multiple files, without having to import them individually
+func ReadDir(ctx context.Context, parser *hclparse.Parser) (*ci.Pipeline, hcl.Diagnostics) {
+	var diags hcl.Diagnostics
+	dir := ConfigFileDir(ctx)
+	togomakFiles, err := os.ReadDir(dir)
+	if err != nil {
+		return nil, diags.Append(&hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  "directory not found",
+			Detail:   err.Error(),
+		})
+	}
+	var pipes []*pipelineMeta
+	for _, file := range togomakFiles {
+		if file.IsDir() {
+			continue
+		}
+		if !strings.HasSuffix(file.Name(), ".hcl") {
+			continue
+		}
+		f, d := parser.ParseHCLFile(filepath.Join(dir, file.Name()))
+		diags = diags.Extend(d)
+
+		p := &ci.Pipeline{}
+
+		d = gohcl.DecodeBody(f.Body, nil, p)
+		diags = diags.Extend(d)
+		pipes = append(pipes, &pipelineMeta{
+			pipe:     p,
+			f:        f,
+			filename: file.Name(),
+		})
+
+	}
+	return createRawPipeline(pipes...)
+
+}
+
+// pipelineMeta is a helper struct to create a pipeline from multiple pipelines
+// this additionally includes the file pointer f, and the filename
+type pipelineMeta struct {
+	pipe     *ci.Pipeline
+	f        *hcl.File
+	filename string
+}
+
+// createRawPipeline creates a pipeline from multiple pipelines. This is useful when you want to merge multiple
+// pipelines together, without having to import them individually
+func createRawPipeline(pipelines ...*pipelineMeta) (*ci.Pipeline, hcl.Diagnostics) {
+	pipe := &ci.Pipeline{}
+
+	var diags hcl.Diagnostics
+
+	var versionDefinedFromFilename string
+	for _, p := range pipelines {
+		if pipe.Builder.Version == 0 && p.pipe.Builder.Version != 0 {
+			pipe.Builder.Version = p.pipe.Builder.Version
+			versionDefinedFromFilename = p.filename
+		}
+		if p.pipe.Builder.Version != pipe.Builder.Version && p.pipe.Builder.Version != 0 {
+			// when overriding and using multiple pipelines, the version of the togomak pipeline schema is
+			// required to be the same
+			return nil, diags.Append(&hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  "version mismatch",
+				Detail:   fmt.Sprintf("version mismatch between pipelines: %d (%s) and %d (%s)", p.pipe.Builder.Version, p.filename, pipe.Builder.Version, versionDefinedFromFilename),
+			})
+		}
+
+		// TODO: create an error if there are duplicate resource definition
+		pipe.Stages = append(pipe.Stages, p.pipe.Stages...)
+		pipe.Data = append(pipe.Data, p.pipe.Data...)
+		pipe.DataProviders = append(pipe.DataProviders, p.pipe.DataProviders...)
+		pipe.Macros = append(pipe.Macros, p.pipe.Macros...)
+		pipe.Locals = append(pipe.Locals, p.pipe.Locals...)
+		pipe.Local = append(pipe.Local, p.pipe.Local...)
+	}
+	return pipe, diags
+}