Make sure log groups are created before and deleted after lambdas (#63)

.editorconfig

@@ -11,6 +11,10 @@ trim_trailing_whitespace = true
 indent_size = 2
 indent_style = space
 
+[*.go]
+indent_size = 2
+indent_style = tab
+
 [*.md]
 max_line_length = 0
 trim_trailing_whitespace = false

README.md

@@ -220,12 +220,12 @@ Available targets:
 | Name | Source | Version |
 |------|--------|---------|
 | <a name="module_cloudwatch_event"></a> [cloudwatch\_event](#module\_cloudwatch\_event) | cloudposse/cloudwatch-events/aws | 0.6.1 |
-| <a name="module_forwarder_log_artifact"></a> [forwarder\_log\_artifact](#module\_forwarder\_log\_artifact) | cloudposse/module-artifact/external | 0.7.2 |
+| <a name="module_forwarder_log_artifact"></a> [forwarder\_log\_artifact](#module\_forwarder\_log\_artifact) | cloudposse/module-artifact/external | 0.8.0 |
 | <a name="module_forwarder_log_label"></a> [forwarder\_log\_label](#module\_forwarder\_log\_label) | cloudposse/label/null | 0.25.0 |
 | <a name="module_forwarder_log_s3_label"></a> [forwarder\_log\_s3\_label](#module\_forwarder\_log\_s3\_label) | cloudposse/label/null | 0.25.0 |
-| <a name="module_forwarder_rds_artifact"></a> [forwarder\_rds\_artifact](#module\_forwarder\_rds\_artifact) | cloudposse/module-artifact/external | 0.7.2 |
+| <a name="module_forwarder_rds_artifact"></a> [forwarder\_rds\_artifact](#module\_forwarder\_rds\_artifact) | cloudposse/module-artifact/external | 0.8.0 |
 | <a name="module_forwarder_rds_label"></a> [forwarder\_rds\_label](#module\_forwarder\_rds\_label) | cloudposse/label/null | 0.25.0 |
-| <a name="module_forwarder_vpclogs_artifact"></a> [forwarder\_vpclogs\_artifact](#module\_forwarder\_vpclogs\_artifact) | cloudposse/module-artifact/external | 0.7.2 |
+| <a name="module_forwarder_vpclogs_artifact"></a> [forwarder\_vpclogs\_artifact](#module\_forwarder\_vpclogs\_artifact) | cloudposse/module-artifact/external | 0.8.0 |
 | <a name="module_forwarder_vpclogs_label"></a> [forwarder\_vpclogs\_label](#module\_forwarder\_vpclogs\_label) | cloudposse/label/null | 0.25.0 |
 | <a name="module_this"></a> [this](#module\_this) | cloudposse/label/null | 0.25.0 |
 

docs/terraform.md

@@ -19,12 +19,12 @@
 | Name | Source | Version |
 |------|--------|---------|
 | <a name="module_cloudwatch_event"></a> [cloudwatch\_event](#module\_cloudwatch\_event) | cloudposse/cloudwatch-events/aws | 0.6.1 |
-| <a name="module_forwarder_log_artifact"></a> [forwarder\_log\_artifact](#module\_forwarder\_log\_artifact) | cloudposse/module-artifact/external | 0.7.2 |
+| <a name="module_forwarder_log_artifact"></a> [forwarder\_log\_artifact](#module\_forwarder\_log\_artifact) | cloudposse/module-artifact/external | 0.8.0 |
 | <a name="module_forwarder_log_label"></a> [forwarder\_log\_label](#module\_forwarder\_log\_label) | cloudposse/label/null | 0.25.0 |
 | <a name="module_forwarder_log_s3_label"></a> [forwarder\_log\_s3\_label](#module\_forwarder\_log\_s3\_label) | cloudposse/label/null | 0.25.0 |
-| <a name="module_forwarder_rds_artifact"></a> [forwarder\_rds\_artifact](#module\_forwarder\_rds\_artifact) | cloudposse/module-artifact/external | 0.7.2 |
+| <a name="module_forwarder_rds_artifact"></a> [forwarder\_rds\_artifact](#module\_forwarder\_rds\_artifact) | cloudposse/module-artifact/external | 0.8.0 |
 | <a name="module_forwarder_rds_label"></a> [forwarder\_rds\_label](#module\_forwarder\_rds\_label) | cloudposse/label/null | 0.25.0 |
-| <a name="module_forwarder_vpclogs_artifact"></a> [forwarder\_vpclogs\_artifact](#module\_forwarder\_vpclogs\_artifact) | cloudposse/module-artifact/external | 0.7.2 |
+| <a name="module_forwarder_vpclogs_artifact"></a> [forwarder\_vpclogs\_artifact](#module\_forwarder\_vpclogs\_artifact) | cloudposse/module-artifact/external | 0.8.0 |
 | <a name="module_forwarder_vpclogs_label"></a> [forwarder\_vpclogs\_label](#module\_forwarder\_vpclogs\_label) | cloudposse/label/null | 0.25.0 |
 | <a name="module_this"></a> [this](#module\_this) | cloudposse/label/null | 0.25.0 |
 

lambda-log.tf

@@ -38,7 +38,7 @@ module "forwarder_log_artifact" {
   count = local.lambda_enabled && var.forwarder_log_enabled ? 1 : 0
 
   source  = "cloudposse/module-artifact/external"
-  version = "0.7.2"
+  version = "0.8.0"
 
   filename    = "forwarder-log.zip"
   module_name = var.dd_module_name
@@ -55,6 +55,9 @@ resource "aws_iam_role" "lambda_forwarder_log" {
   assume_role_policy   = data.aws_iam_policy_document.assume_role[0].json
   permissions_boundary = var.log_permissions_boundary
   tags                 = module.forwarder_log_label.tags
+
+  # AWS will create the log group if needed. Make sure we create it first.
+  depends_on = [aws_cloudwatch_log_group.forwarder_log]
 }
 
 resource "aws_iam_policy" "lambda_forwarder_log" {
@@ -111,6 +114,9 @@ resource "aws_lambda_function" "forwarder_log" {
   }
 
   tags = module.forwarder_log_label.tags
+
+  # AWS will create the log group if needed. Make sure we create it first.
+  depends_on = [aws_cloudwatch_log_group.forwarder_log]
 }
 
 resource "aws_lambda_permission" "allow_s3_bucket" {
@@ -202,7 +208,7 @@ resource "aws_iam_role_policy_attachment" "datadog_s3" {
 resource "aws_cloudwatch_log_group" "forwarder_log" {
   count = local.lambda_enabled && var.forwarder_log_enabled ? 1 : 0
 
-  name              = "/aws/lambda/${aws_lambda_function.forwarder_log[0].function_name}"
+  name              = "/aws/lambda/${module.forwarder_log_label.id}"
   retention_in_days = var.forwarder_log_retention_days
 
   kms_key_id = var.kms_key_id

lambda-rds.tf

@@ -24,7 +24,7 @@ module "forwarder_rds_artifact" {
   count = local.lambda_enabled && var.forwarder_rds_enabled ? 1 : 0
 
   source  = "cloudposse/module-artifact/external"
-  version = "0.7.2"
+  version = "0.8.0"
 
   filename    = "forwarder-rds.py"
   module_name = var.dd_module_name
@@ -50,6 +50,9 @@ resource "aws_iam_role" "lambda_forwarder_rds" {
   assume_role_policy   = data.aws_iam_policy_document.assume_role[0].json
   permissions_boundary = var.rds_permissions_boundary
   tags                 = module.forwarder_rds_label.tags
+
+  # AWS will create the log group if needed. Make sure we create it first.
+  depends_on = [aws_cloudwatch_log_group.forwarder_rds]
 }
 
 resource "aws_iam_policy" "lambda_forwarder_rds" {
@@ -106,6 +109,9 @@ resource "aws_lambda_function" "forwarder_rds" {
   }
 
   tags = module.forwarder_rds_label.tags
+
+  # AWS will create the log group if needed. Make sure we create it first.
+  depends_on = [aws_cloudwatch_log_group.forwarder_rds]
 }
 
 resource "aws_lambda_permission" "cloudwatch_enhanced_rds_monitoring" {
@@ -130,7 +136,7 @@ resource "aws_cloudwatch_log_subscription_filter" "datadog_log_subscription_filt
 resource "aws_cloudwatch_log_group" "forwarder_rds" {
   count = local.lambda_enabled && var.forwarder_rds_enabled ? 1 : 0
 
-  name              = "/aws/lambda/${aws_lambda_function.forwarder_rds[0].function_name}"
+  name              = "/aws/lambda/${module.forwarder_rds_label.id}"
   retention_in_days = var.forwarder_log_retention_days
   kms_key_id        = var.kms_key_id
 

lambda-vpc-logs.tf

@@ -23,7 +23,7 @@ module "forwarder_vpclogs_artifact" {
   count = local.lambda_enabled && var.forwarder_vpc_logs_enabled ? 1 : 0
 
   source  = "cloudposse/module-artifact/external"
-  version = "0.7.2"
+  version = "0.8.0"
 
   filename    = "lambda_function.py"
   module_name = var.dd_module_name
@@ -49,6 +49,9 @@ resource "aws_iam_role" "lambda_forwarder_vpclogs" {
   assume_role_policy   = data.aws_iam_policy_document.assume_role[0].json
   permissions_boundary = var.vpc_logs_permissions_boundary
   tags                 = module.forwarder_vpclogs_label.tags
+
+  # AWS will create the log group if needed. Make sure we create it first.
+  depends_on = [aws_cloudwatch_log_group.forwarder_vpclogs]
 }
 
 resource "aws_iam_policy" "lambda_forwarder_vpclogs" {
@@ -104,9 +107,10 @@ resource "aws_lambda_function" "forwarder_vpclogs" {
     mode = var.tracing_config_mode
   }
 
-
-
   tags = module.forwarder_vpclogs_label.tags
+
+  # AWS will create the log group if needed. Make sure we create it first.
+  depends_on = [aws_cloudwatch_log_group.forwarder_vpclogs]
 }
 
 resource "aws_lambda_permission" "cloudwatch_vpclogs" {
@@ -131,7 +135,7 @@ resource "aws_cloudwatch_log_subscription_filter" "datadog_log_subscription_filt
 resource "aws_cloudwatch_log_group" "forwarder_vpclogs" {
   count = local.lambda_enabled && var.forwarder_vpc_logs_enabled ? 1 : 0
 
-  name              = "/aws/lambda/${aws_lambda_function.forwarder_vpclogs[0].function_name}"
+  name              = "/aws/lambda/${module.forwarder_vpclogs_label.id}"
   retention_in_days = var.forwarder_log_retention_days
   kms_key_id        = var.kms_key_id
 

main.tf

@@ -32,8 +32,7 @@ locals {
   # If map is supplied, merge map with context, or use only context
   # Convert map to dd tags equivalent
   dd_tags = length(var.dd_tags_map) > 0 ? [
-    for tagk, tagv in var.dd_tags_map :
-    tagv != null ? format("%s:%s", tagk, tagv) : tagk
+    for tagk, tagv in var.dd_tags_map : (tagv != null ? format("%s:%s", tagk, tagv) : tagk)
   ] : var.dd_tags
   dd_tags_env = { DD_TAGS = join(",", local.dd_tags) }
 

test/src/Makefile

@@ -1,9 +1,8 @@
-export TF_CLI_ARGS_init ?= -get-plugins=true
-export TERRAFORM_VERSION ?= $(shell curl -s https://checkpoint-api.hashicorp.com/v1/check/terraform | jq -r -M '.current_version' | cut -d. -f1-2)
+export TERRAFORM_VERSION ?= $(shell curl -s https://checkpoint-api.hashicorp.com/v1/check/terraform | jq -r -M '.current_version' | cut -d. -f1)
 
 .DEFAULT_GOAL : all
-.PHONY: all
 
+.PHONY: all
 ## Default target
 all: test
 
@@ -15,10 +14,8 @@ init:
 .PHONY : test
 ## Run tests
 test: init
-	# This project runs `git` externally, so it needs extra permissions when run by a GitHub Action
-	[[ -n "$$GITHUB_WORKSPACE" ]] && git config --global --add safe.directory "$$GITHUB_WORKSPACE" || true
 	go mod download
-	go test -v -timeout 30m
+	go test -v -timeout 10m
 
 ## Run tests in docker container
 docker/test:

test/src/examples_complete_test.go

@@ -1,32 +1,48 @@
 package test
 
 import (
+	"os/exec"
 	"regexp"
 	"strings"
 	"testing"
 
 	"github.com/gruntwork-io/terratest/modules/random"
 	"github.com/gruntwork-io/terratest/modules/terraform"
-	testStructure "github.com/gruntwork-io/terratest/modules/test-structure"
 	"github.com/stretchr/testify/assert"
 	"k8s.io/apimachinery/pkg/util/runtime"
 )
 
 // Test the Terraform module in examples/complete using Terratest.
 func TestExamplesComplete(t *testing.T) {
-	t.Parallel()
+	// This module needs to be run inside a Git Repository, so we cannot run it in parallel
+	// t.Parallel()
+
+	// If running on a GitHub Action Runner, invoke the necessary blessing
+	cmd := exec.Command("bash", "-c", "if [[ -d /__w/actions/actions ]]; then git config --global --add safe.directory /__w/actions/actions; fi")
+	var stdout strings.Builder
+	cmd.Stdout = &stdout
+	var stderr strings.Builder
+	cmd.Stderr = &stderr
+
+	if err := cmd.Run(); err != nil {
+		t.Logf("Running command: %s", cmd.String())
+		t.Logf("command stdout: %s", stdout.String())
+		t.Logf("command stderr: %s", stderr.String())
+		t.Log(err)
+	} else if stdout.Len() > 0 || stderr.Len() > 0 {
+		t.Logf("Running command: %s", cmd.String())
+		t.Logf("command stdout: %s", stdout.String())
+		t.Logf("command stderr: %s", stderr.String())
+	}
+
 	randID := strings.ToLower(random.UniqueId())
 	attributes := []string{randID}
 
-	rootFolder := "../../"
-	terraformFolderRelativeToRoot := "examples/complete"
 	varFiles := []string{"fixtures.us-east-2.tfvars"}
 
-	tempTestFolder := testStructure.CopyTerraformFolderToTemp(t, rootFolder, terraformFolderRelativeToRoot)
-
 	terraformOptions := &terraform.Options{
 		// The path to where our Terraform code is located
-		TerraformDir: tempTestFolder,
+		TerraformDir: "../../examples/complete",
 		Upgrade:      true,
 		// Variables to pass to our Terraform code using -var-file options
 		VarFiles: varFiles,
@@ -36,11 +52,11 @@ func TestExamplesComplete(t *testing.T) {
 	}
 
 	// At the end of the test, run `terraform destroy` to clean up any resources that were created
-	defer cleanup(t, terraformOptions, tempTestFolder)
+	defer terraform.Destroy(t, terraformOptions)
 
 	// If Go runtime crushes, run `terraform destroy` to clean up any resources that were created
 	defer runtime.HandleCrash(func(i interface{}) {
-		cleanup(t, terraformOptions, tempTestFolder)
+		defer terraform.Destroy(t, terraformOptions)
 	})
 
 	// This will run `terraform init` and `terraform apply` and fail the test if there are any errors
@@ -51,19 +67,17 @@ func TestExamplesComplete(t *testing.T) {
 }
 
 func TestExamplesCompleteDisabled(t *testing.T) {
-	t.Parallel()
+	// This module needs to be run inside a Git Repository, so we cannot run it in parallel
+	// t.Parallel()
+
 	randID := strings.ToLower(random.UniqueId())
 	attributes := []string{randID}
 
-	rootFolder := "../../"
-	terraformFolderRelativeToRoot := "examples/complete"
 	varFiles := []string{"fixtures.us-east-2.tfvars"}
 
-	tempTestFolder := testStructure.CopyTerraformFolderToTemp(t, rootFolder, terraformFolderRelativeToRoot)
-
 	terraformOptions := &terraform.Options{
 		// The path to where our Terraform code is located
-		TerraformDir: tempTestFolder,
+		TerraformDir: "../../examples/complete",
 		Upgrade:      true,
 		// Variables to pass to our Terraform code using -var-file options
 		VarFiles: varFiles,
@@ -74,7 +88,12 @@ func TestExamplesCompleteDisabled(t *testing.T) {
 	}
 
 	// At the end of the test, run `terraform destroy` to clean up any resources that were created
-	defer cleanup(t, terraformOptions, tempTestFolder)
+	defer terraform.Destroy(t, terraformOptions)
+
+	// If Go runtime crushes, run `terraform destroy` to clean up any resources that were created
+	defer runtime.HandleCrash(func(i interface{}) {
+		defer terraform.Destroy(t, terraformOptions)
+	})
 
 	// This will run `terraform init` and `terraform apply` and fail the test if there are any errors
 	results := terraform.InitAndApply(t, terraformOptions)