[feature] Add GitHub Webhooks archiver and S3 private bucket modules (#…

…112)

.gitignore

@@ -1,4 +1,5 @@
 .terraform
 *.tfstate
 *.tfstate.backup
-bin
+bin
+.idea

README.md

@@ -92,6 +92,13 @@ This creates a security-audit role, based off the AWS-managed policy, but with a
 
 [Read More](aws-iam-role-security-audit/README.md)
 
+### GitHub Webhooks to S3
+
+Accept GitHub webhooks and store them in S3
+
+[Read More](github-webhooks-to-s3/README.md)
+
+
 ## Contributing
 
 ### Writing tests

aws-s3-private-bucket/README.md

@@ -0,0 +1,22 @@
+<!-- START -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|:----:|:-----:|:-----:|
+| bucket\_name |  | string | n/a | yes |
+| bucket\_policy |  | string | `""` | no |
+| env |  | string | n/a | yes |
+| owner |  | string | n/a | yes |
+| project |  | string | n/a | yes |
+| service |  | string | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| arn |  |
+| domain\_name |  |
+| id |  |
+| name | We do this to hint TF dependency graph since modules can't depend_on |
+
+<!-- END -->

aws-s3-private-bucket/main.tf

@@ -0,0 +1,66 @@
+locals {
+  tags = {
+    project   = "${var.project}"
+    env       = "${var.env}"
+    service   = "${var.service}"
+    owner     = "${var.owner}"
+    managedBy = "terraform"
+  }
+}
+
+resource "aws_s3_bucket" "bucket" {
+  bucket = "${var.bucket_name}"
+  acl    = "private"
+
+  policy = "${data.aws_iam_policy_document.bucket_policy.json}"
+
+  versioning {
+    enabled = true
+  }
+
+  # TODO
+  #   logging {
+  #   target_bucket = ""
+  #   target_prefix = ""
+  # }
+
+  server_side_encryption_configuration {
+    rule {
+      apply_server_side_encryption_by_default {
+        sse_algorithm = "AES256"
+      }
+    }
+  }
+  tags = "${local.tags}"
+}
+
+resource "aws_s3_bucket_public_access_block" "bucket" {
+  bucket = "${aws_s3_bucket.bucket.id}"
+
+  block_public_acls   = true
+  block_public_policy = true
+}
+
+data "aws_iam_policy_document" "bucket_policy" {
+  # Deny access to bucket if it's not accessed through HTTPS
+  source_json = "${var.bucket_policy}"
+
+  statement {
+    sid       = "EnforceHTTPS"
+    actions   = ["s3:GetObject"]
+    resources = ["arn:aws:s3:::${var.bucket_name}/*"]
+
+    principals {
+      type        = "*"
+      identifiers = ["*"]
+    }
+
+    effect = "Deny"
+
+    condition {
+      test     = "Bool"
+      variable = "aws:SecureTransport"
+      values   = ["false"]
+    }
+  }
+}

aws-s3-private-bucket/module_test.go

@@ -0,0 +1,14 @@
+package test
+
+import (
+	"testing"
+
+	"github.com/gruntwork-io/terratest/modules/terraform"
+)
+
+func TestPrivateBucket(t *testing.T) {
+	options := &terraform.Options{
+		TerraformDir: ".",
+	}
+	terraform.Init(t, options)
+}

aws-s3-private-bucket/outputs.tf

@@ -0,0 +1,16 @@
+// HACK(el): we do this to hint TF dependency graph since modules can't depend_on
+output "name" {
+  value = "${var.bucket_name.id}"
+}
+
+output "domain_name" {
+  value = "${aws_s3_bucket.bucket.bucket_domain_name}"
+}
+
+output "arn" {
+  value = "${aws_s3_bucket.bucket.arn}"
+}
+
+output "id" {
+  value = "${aws_s3_bucket.bucket.id}"
+}

aws-s3-private-bucket/variables.tf

@@ -0,0 +1,24 @@
+variable "bucket_name" {
+  type = "string"
+}
+
+variable "bucket_policy" {
+  type    = "string"
+  default = ""
+}
+
+variable "project" {
+  type = "string"
+}
+
+variable "env" {
+  type = "string"
+}
+
+variable "service" {
+  type = "string"
+}
+
+variable "owner" {
+  type = "string"
+}

github-webhooks-to-s3/README.md

@@ -0,0 +1,24 @@
+<!-- START -->
+Accept GitHub webhooks to S3. Keeps track of events such as pushing code to a repository.
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|:----:|:-----:|:-----:|
+| certificate\_arn | A certificate in us-east-1 for var.fqdn | string | n/a | yes |
+| env | Env for tagging and naming. | string | n/a | yes |
+| fqdn | The fqdn to expose the api gateway as | string | n/a | yes |
+| iam\_path |  | string | `"/"` | no |
+| lambda\_source\_s3\_bucket | The s3 bucket where to find the lambda executable | string | `"shared-infra-prod-assets"` | no |
+| lambda\_source\_s3\_key | The s3 key where to find the lambda executable | string | `"go-misc/lambdas/2019/06/03/github_to_firehose.zip"` | no |
+| owner | Owner for tagging and naming. | string | n/a | yes |
+| project | Project for tagging and naming. | string | n/a | yes |
+| route53\_zone\_id | The route53 zone id for fqdn's domain | string | n/a | yes |
+| service | Service for tagging and naming. | string | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+
+<!-- END -->

github-webhooks-to-s3/api_gateway.tf

@@ -0,0 +1,64 @@
+// https://learn.hashicorp.com/terraform/aws/lambda-api-gateway
+resource "aws_api_gateway_rest_api" "github" {
+  name        = "${local.name}"
+  description = "Github webhook ingestion"
+}
+
+resource "aws_api_gateway_resource" "github" {
+  rest_api_id = "${aws_api_gateway_rest_api.github.id}"
+  parent_id   = "${aws_api_gateway_rest_api.github.root_resource_id}"
+  path_part   = "{proxy+}"
+}
+
+resource "aws_api_gateway_method" "github" {
+  rest_api_id   = "${aws_api_gateway_rest_api.github.id}"
+  resource_id   = "${aws_api_gateway_resource.github.id}"
+  http_method   = "POST"
+  authorization = "NONE"
+}
+
+resource "aws_api_gateway_integration" "lambda" {
+  rest_api_id = "${aws_api_gateway_rest_api.github.id}"
+  resource_id = "${aws_api_gateway_method.github.resource_id}"
+  http_method = "${aws_api_gateway_method.github.http_method}"
+
+  integration_http_method = "POST"
+  type                    = "AWS_PROXY"
+  uri                     = "${aws_lambda_function.lambda.invoke_arn}"
+}
+
+resource "aws_api_gateway_method" "github_root" {
+  rest_api_id   = "${aws_api_gateway_rest_api.github.id}"
+  resource_id   = "${aws_api_gateway_rest_api.github.root_resource_id}"
+  http_method   = "ANY"
+  authorization = "NONE"
+}
+
+resource "aws_api_gateway_integration" "lambda_root" {
+  rest_api_id = "${aws_api_gateway_rest_api.github.id}"
+  resource_id = "${aws_api_gateway_method.github_root.resource_id}"
+  http_method = "${aws_api_gateway_method.github_root.http_method}"
+
+  integration_http_method = "POST"
+  type                    = "AWS_PROXY"
+  uri                     = "${aws_lambda_function.lambda.invoke_arn}"
+}
+
+resource "aws_api_gateway_deployment" "github" {
+  depends_on = [
+    "aws_api_gateway_integration.lambda",
+    "aws_api_gateway_integration.lambda_root",
+  ]
+
+  rest_api_id = "${aws_api_gateway_rest_api.github.id}"
+  stage_name  = "${var.env}"
+}
+
+resource "aws_lambda_permission" "apigw" {
+  statement_id  = "AllowAPIGatewayInvoke"
+  action        = "lambda:InvokeFunction"
+  function_name = "${aws_lambda_function.lambda.arn}"
+  principal     = "apigateway.amazonaws.com"
+
+  source_arn = "${aws_api_gateway_deployment.github.execution_arn}/*/*"
+}

github-webhooks-to-s3/certs.tf

@@ -0,0 +1,34 @@
+resource "aws_api_gateway_domain_name" "github" {
+  certificate_arn = "${var.certificate_arn}"
+  domain_name     = "${var.fqdn}"
+}
+
+resource "aws_api_gateway_base_path_mapping" "github" {
+  api_id      = "${aws_api_gateway_rest_api.github.id}"
+  stage_name  = "${aws_api_gateway_deployment.github.stage_name}"
+  domain_name = "${aws_api_gateway_domain_name.github.domain_name}"
+}
+
+resource "aws_route53_record" "github" {
+  name    = "${aws_api_gateway_domain_name.github.domain_name}"
+  type    = "A"
+  zone_id = "${var.route53_zone_id}"
+
+  alias {
+    evaluate_target_health = true
+    name                   = "${aws_api_gateway_domain_name.github.cloudfront_domain_name}"
+    zone_id                = "${aws_api_gateway_domain_name.github.cloudfront_zone_id}"
+  }
+}
+
+resource "aws_route53_record" "github-ipv6" {
+  name    = "${aws_api_gateway_domain_name.github.domain_name}"
+  type    = "AAAA"
+  zone_id = "${var.route53_zone_id}"
+
+  alias {
+    evaluate_target_health = true
+    name                   = "${aws_api_gateway_domain_name.github.cloudfront_domain_name}"
+    zone_id                = "${aws_api_gateway_domain_name.github.cloudfront_zone_id}"
+  }
+}

github-webhooks-to-s3/firehose.tf

@@ -0,0 +1,102 @@
+module "bucket" {
+  source = "../aws-s3-private-bucket"
+
+  bucket_name   = "${local.name}"
+  bucket_policy = ""
+
+  project = "${var.project}"
+  env     = "${var.env}"
+  service = "${var.service}"
+  owner   = "${var.owner}"
+}
+
+data "aws_iam_policy_document" "firehose" {
+  statement {
+    sid     = "EnableFirehoseAssumeRole"
+    effect  = "Allow"
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["firehose.amazonaws.com"]
+    }
+  }
+}
+
+resource "aws_iam_role" "firehose" {
+  name = "${local.name}-firehose"
+
+  assume_role_policy = "${data.aws_iam_policy_document.firehose.json}"
+  tags               = "${local.tags}"
+}
+
+data "aws_iam_policy_document" "firehose-to-s3" {
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "s3:AbortMultipartUpload",
+      "s3:GetBucketLocation",
+      "s3:GetObject",
+      "s3:ListBucket",
+      "s3:ListBucketMultipartUploads",
+      "s3:PutObject",
+    ]
+
+    resources = [
+      "${module.bucket.arn}",
+      "${module.bucket.arn}/*",
+    ]
+  }
+
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "logs:CreateLogStream",
+      "logs:PutLogEvents",
+      "logs:GetLogEvents",
+    ]
+
+    resources = [
+      "${aws_cloudwatch_log_group.firehose.arn}",
+      "${aws_cloudwatch_log_group.firehose.arn}/*",
+    ]
+  }
+}
+
+resource "aws_iam_role_policy" "firehose-s3" {
+  name   = "firehose-s3"
+  role   = "${aws_iam_role.firehose.id}"
+  policy = "${data.aws_iam_policy_document.firehose-to-s3.json}"
+}
+
+resource "aws_kinesis_firehose_delivery_stream" "firehose" {
+  name        = "${local.name}"
+  destination = "s3"
+
+  s3_configuration {
+    role_arn           = "${aws_iam_role.firehose.arn}"
+    bucket_arn         = "${module.bucket.arn}"
+    prefix             = ""
+    compression_format = "GZIP"
+
+    cloudwatch_logging_options {
+      enabled         = true
+      log_group_name  = "${aws_cloudwatch_log_group.firehose.name}"
+      log_stream_name = "${aws_cloudwatch_log_stream.firehose.name}"
+    }
+  }
+
+  tags = "${local.tags}"
+}
+
+resource "aws_cloudwatch_log_group" "firehose" {
+  name = "${local.name}-firehose"
+  tags = "${local.tags}"
+}
+
+resource "aws_cloudwatch_log_stream" "firehose" {
+  name           = "status"
+  log_group_name = "${aws_cloudwatch_log_group.firehose.name}"
+}