Fix multiple race conditions and error in module inputs

cmd/togomak/main.go

@@ -8,6 +8,7 @@ import (
 	"github.com/srevinsaju/togomak/v1/internal/cache"
 	"github.com/srevinsaju/togomak/v1/internal/ci"
 	"github.com/srevinsaju/togomak/v1/internal/filter"
+	"github.com/srevinsaju/togomak/v1/internal/global"
 	"github.com/srevinsaju/togomak/v1/internal/meta"
 	"github.com/srevinsaju/togomak/v1/internal/orchestra"
 	"github.com/srevinsaju/togomak/v1/internal/path"
@@ -260,6 +261,8 @@ func newConfigFromCliContext(ctx *cli.Context) ci.ConductorConfig {
 
 func run(ctx *cli.Context) error {
 	cfg := newConfigFromCliContext(ctx)
+	logger := ci.NewLogger(cfg)
+	global.SetLogger(logger)
 
 	t := ci.NewConductor(cfg)
 	v := orchestra.Perform(t)

examples/module-local/calculator/togomak.hcl

@@ -0,0 +1,22 @@
+togomak {
+  version = 2
+}
+
+variable "a" {
+  type = number
+  description = "first variable"
+}
+variable "b" {
+  type = number 
+  description = "second variable"
+}
+
+variable "operation" {
+  type = string 
+  description = "Operation to perform, any of: [add, subtract, multiply, divide]"
+}
+
+stage "add" {
+  if = var.operation == "add"
+  script = "echo ${var.a} + ${var.b} is ${var.a + var.b}"
+}

examples/module-local/togomak.hcl

@@ -0,0 +1,16 @@
+ togomak {
+  version = 2
+}
+
+locals {
+  a = 99
+  b = 1
+}
+
+module "add" {
+  source = "./calculator"
+  a = local.a
+  b = local.b
+  operation = "add"
+}
+

go.mod

@@ -139,3 +139,5 @@ require (
 replace github.com/sirupsen/logrus v1.9.0 => github.com/srevinsaju/logrus v1.10.4-0.20231021232453-c286e2e09f97
 
 replace github.com/sirupsen/logrus v1.9.2 => github.com/srevinsaju/logrus v1.10.4-0.20231021232453-c286e2e09f97
+
+replace github.com/hashicorp/go-getter => github.com/srevinsaju/go-getter v0.0.0-20231023111034-c5f284a1f9c8

go.sum

@@ -431,8 +431,6 @@ github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9n
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
 github.com/hashicorp/go-envparse v0.1.0 h1:bE++6bhIsNCPLvgDZkYqo3nA+/PFI51pkrHdmPSDFPY=
 github.com/hashicorp/go-envparse v0.1.0/go.mod h1:OHheN1GoygLlAkTlXLXvAdnXdZxy8JUweQ1rAXx1xnc=
-github.com/hashicorp/go-getter v1.7.2 h1:uJDtyXwEfalmp1PqdxuhZqrNkUyClZAhVeZYTArbqkg=
-github.com/hashicorp/go-getter v1.7.2/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744=
 github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo=
 github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I=
 github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
@@ -575,6 +573,8 @@ github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:s
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM=
 github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ=
+github.com/srevinsaju/go-getter v0.0.0-20231023111034-c5f284a1f9c8 h1:+tbqxizAn9s3GMNKaBXCAy5Is6dUSTdsSL49Y/dt0vY=
+github.com/srevinsaju/go-getter v0.0.0-20231023111034-c5f284a1f9c8/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744=
 github.com/srevinsaju/logrus v1.10.4-0.20231021232453-c286e2e09f97 h1:ED4yRDEtRpuqxvXHR2l9g5G240YXWKHEye+SDtwrf6A=
 github.com/srevinsaju/logrus v1.10.4-0.20231021232453-c286e2e09f97/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

internal/blocks/data/git.go

@@ -6,9 +6,7 @@ import (
 	"context"
 	"fmt"
 	"github.com/hashicorp/hcl/v2"
-	"github.com/sirupsen/logrus"
 	"github.com/srevinsaju/togomak/v1/internal/conductor"
-	"github.com/srevinsaju/togomak/v1/internal/global"
 	"github.com/srevinsaju/togomak/v1/internal/ui"
 	"github.com/zclconf/go-cty/cty"
 	"net/url"
@@ -69,10 +67,6 @@ type GitProvider struct {
 	cfg gitProviderConfig
 }
 
-func (e *GitProvider) Logger() *logrus.Entry {
-	return global.Logger().WithField("provider", e.Name())
-}
-
 func (e *GitProvider) Name() string {
 	return "git"
 }
@@ -318,7 +312,7 @@ func (e *GitProvider) Value(conductor conductor.Conductor, ctx context.Context,
 }
 
 func (e *GitProvider) Attributes(conductor conductor.Conductor, ctx context.Context, id string, opts ...ProviderOption) (map[string]cty.Value, hcl.Diagnostics) {
-	logger := e.Logger()
+	logger := conductor.Logger().WithField("data", e.Name())
 	var diags hcl.Diagnostics
 	if !e.initialized {
 		panic("provider not initialized")
@@ -348,7 +342,7 @@ func (e *GitProvider) Attributes(conductor conductor.Conductor, ctx context.Cont
 	}
 	// TODO: implement git submodules
 
-	destination, d := e.resolveDestination(ctx, id)
+	destination, d := e.resolveDestination(conductor, ctx, id)
 	diags = diags.Extend(d)
 	if diags.HasErrors() {
 		return nil, diags
@@ -514,9 +508,9 @@ func (e *GitProvider) Attributes(conductor conductor.Conductor, ctx context.Cont
 	return attrs, diags
 }
 
-func (e *GitProvider) resolveDestination(ctx context.Context, id string) (string, hcl.Diagnostics) {
-	logger := e.Logger()
-	tmpDir := global.TempDir()
+func (e *GitProvider) resolveDestination(conductor conductor.Conductor, ctx context.Context, id string) (string, hcl.Diagnostics) {
+	logger := conductor.Logger().WithField("data", e.Name())
+	tmpDir := conductor.TempDir()
 
 	var diags hcl.Diagnostics
 	destination := e.cfg.destination

internal/blocks/data/prompt.go

@@ -5,9 +5,7 @@ import (
 	"fmt"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/hashicorp/hcl/v2"
-	"github.com/sirupsen/logrus"
 	"github.com/srevinsaju/togomak/v1/internal/conductor"
-	"github.com/srevinsaju/togomak/v1/internal/global"
 	"github.com/srevinsaju/togomak/v1/internal/meta"
 	"github.com/zclconf/go-cty/cty"
 	"os"
@@ -106,10 +104,6 @@ func (e *PromptProvider) Initialized() bool {
 	return e.initialized
 }
 
-func (e *PromptProvider) Logger() *logrus.Entry {
-	return global.Logger().WithField("provider", e.Name())
-}
-
 func (e *PromptProvider) Value(conductor conductor.Conductor, ctx context.Context, id string, opts ...ProviderOption) (string, hcl.Diagnostics) {
 	var diags hcl.Diagnostics
 	if !e.initialized {
@@ -118,7 +112,7 @@ func (e *PromptProvider) Value(conductor conductor.Conductor, ctx context.Contex
 
 	cfg := NewProviderConfig(opts...)
 
-	logger := e.Logger()
+	logger := conductor.Logger().WithField("data", e.Name())
 
 	envVarName := fmt.Sprintf("%s%s__%s", meta.EnvVarPrefix, e.Name(), id)
 	logger.Tracef("checking for environment variable %s", envVarName)

internal/blocks/data/terraform.go

@@ -7,9 +7,7 @@ import (
 	"github.com/hashicorp/go-getter"
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/terraform-exec/tfexec"
-	"github.com/sirupsen/logrus"
 	"github.com/srevinsaju/togomak/v1/internal/conductor"
-	"github.com/srevinsaju/togomak/v1/internal/global"
 	"github.com/srevinsaju/togomak/v1/internal/ui"
 	"github.com/srevinsaju/togomak/v1/internal/x"
 	"github.com/zclconf/go-cty/cty"
@@ -40,10 +38,6 @@ type TfProvider struct {
 	cfg TfProviderConfig
 }
 
-func (e *TfProvider) Logger() *logrus.Entry {
-	return global.Logger().WithField("provider", e.Name())
-}
-
 func (e *TfProvider) Name() string {
 	return "tf"
 }
@@ -149,8 +143,8 @@ func (e *TfProvider) Value(conductor conductor.Conductor, ctx context.Context, i
 }
 
 func (e *TfProvider) Attributes(conductor conductor.Conductor, ctx context.Context, id string, opts ...ProviderOption) (map[string]cty.Value, hcl.Diagnostics) {
-	logger := e.Logger()
-	tmpDir := global.TempDir()
+	logger := conductor.Logger().WithField("data", e.Name())
+	tmpDir := conductor.TempDir()
 	cfg := NewProviderConfig(opts...)
 
 	var diags hcl.Diagnostics

internal/ci/conductor.go

@@ -7,7 +7,6 @@
 	"github.com/hashicorp/hcl/v2/hclparse"
 	"github.com/sirupsen/logrus"
 	"github.com/srevinsaju/togomak/v1/internal/conductor"
-	"github.com/srevinsaju/togomak/v1/internal/global"
 	"github.com/srevinsaju/togomak/v1/internal/meta"
 	"github.com/srevinsaju/togomak/v1/internal/x"
 	"os"
@@ -36,7 +35,7 @@
 	}
 }
 
-func ConductorWithParser(parser *hclparse.Parser) ConductorOption {
+func ConductorWithParser(parser *Parser) ConductorOption {
 	return func(c *Conductor) {
 		c.Parser = parser
 	}
@@ -97,7 +96,7 @@
 	Process Process
 
 	// Parser is the HCL parser
-	Parser *hclparse.Parser
+	Parser *Parser
 
 	// DiagWriter is the HCL diagnostic writer, it is used to write the diagnostics
 	// to os.Stdout
@@ -111,6 +110,27 @@
 	variables Variables
 }
 
+type Parser struct {
+	parser *hclparse.Parser
+	mu     *sync.RWMutex
+}
+
+func (p *Parser) ParseHCLFile(filename string) (*hcl.File, hcl.Diagnostics) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	return p.parser.ParseHCLFile(filename)
+}
+
+func (p *Parser) Files() map[string]*hcl.File {
+	p.mu.RLock()
+	defer p.mu.RUnlock()
+	return p.parser.Files()
+}
+
+func (c *Conductor) TempDir() string {
+	return c.Process.TempDir
+}
+
 func (c *Conductor) Eval() conductor.Eval {
 	return c.eval
 }
@@ -173,7 +193,6 @@
 	// create a temporary directory
 	tempDir, err := os.MkdirTemp("", "togomak")
 	x.Must(err)
-	global.SetTempDir(tempDir)
 
 	return Process{
 		Id:         pipelineId,
@@ -212,19 +231,22 @@
 	diagWriter := hcl.NewDiagnosticTextWriter(os.Stdout, parser.Files(), 0, true)
 
 	logger := NewLogger(cfg)
-	global.SetLogger(logger)
 
 	dir := Chdir(cfg, logger)
-	cfg.Paths.Cwd = dir
-
+	if dir != cfg.Paths.Cwd {
+		cfg.Paths.Cwd = dir
+	}
 	if cfg.Paths.Module == "" {
 		cfg.Paths.Module = cfg.Paths.Cwd
 	}
 
 	process := NewProcess(cfg)
 
 	c := &Conductor{
-		Parser:     parser,
+		Parser: &Parser{
+			parser: parser,
+			mu:     &sync.RWMutex{},
+		},
 		DiagWriter: diagWriter,
 		ctx:        context.Background(),
 		Process:    process,

internal/ci/data_logging.go

@@ -1,10 +1 @@
 package ci
-
-import (
-	"github.com/sirupsen/logrus"
-	"github.com/srevinsaju/togomak/v1/internal/global"
-)
-
-func (s *Data) Logger() *logrus.Entry {
-	return global.Logger().WithField("data", s.Id)
-}

internal/ci/data_run.go

@@ -18,7 +18,7 @@ func (s *Data) Prepare(conductor *Conductor, skip bool, overridden bool) hcl.Dia
 }
 
 func (s *Data) Run(conductor *Conductor, options ...runnable.Option) (diags hcl.Diagnostics) {
-	logger := s.Logger()
+	logger := conductor.Logger().WithField("data", s.Id)
 	logger.Debugf("running %s.%s.%s", DataBlock, s.Provider, s.Id)
 	hclContext := conductor.Eval().Context()
 	ctx := conductor.Context()

internal/ci/graph.go

@@ -5,7 +5,6 @@ import (
 	"github.com/hashicorp/hcl/v2"
 	"github.com/kendru/darwin/go/depgraph"
 	"github.com/srevinsaju/togomak/v1/internal/blocks"
-	"github.com/srevinsaju/togomak/v1/internal/global"
 	"github.com/srevinsaju/togomak/v1/internal/meta"
 	"github.com/srevinsaju/togomak/v1/internal/x"
 )
@@ -41,10 +40,11 @@ func GraphResolve(ctx context.Context, pipe *Pipeline, g *depgraph.Graph, v []hc
 	}
 	return diags
 }
-func GraphTopoSort(ctx context.Context, pipe *Pipeline) (*depgraph.Graph, hcl.Diagnostics) {
+func GraphTopoSort(conductor *Conductor, pipe *Pipeline) (*depgraph.Graph, hcl.Diagnostics) {
 	g := depgraph.New()
 	var diags hcl.Diagnostics
-	logger := global.Logger().WithField("orchestra", "graph")
+	ctx := conductor.Context()
+	logger := conductor.Logger().WithField("orchestra", "graph")
 
 	x.Must(g.DependOn(meta.PreStage, meta.RootStage))
 	x.Must(g.DependOn(meta.PostStage, meta.PreStage))