Docker based deployments over EC2 (Infra - Terraform, Configuration -…

… Ansible) with disaster recovery support (#105)

README.md

@@ -96,6 +96,35 @@ packer build -var="image_repository=your_value" -var="image_tags=[tag1,tag2]" wa
 |[PgRepack](https://github.com/reorg/pg_repack)        | `pg_repack`      |
 |[PG Auto Failover](https://github.com/hapostgres/pg_auto_failover)| `pgautofailover` |
 |[HyperLogLog](https://github.com/citusdata/postgresql-hll)     | `hll`            |
+### AWS 
+WarpSQL provides a streamlined approach to deploying and managing PostgreSQL databases on AWS EC2 instances, complete with a disaster recovery solution powered by Barman. 
+> **Warning**
+WarpSQL is a work in progress, and the current setup allows public SSH access to instances, which might not be secure.
+
+
+To get started, ensure you have your AWS credentials set up and Docker,Terraform installed.
+
+First, create Docker images using the provided Dockerfiles in the `barman` and `warpsql_ssh` directories. Once built, push these images to a public image repository.
+Specify the repository where you pushed these Docker images by passing the repository URL as a variable in the Terraform command.
+
+To launch WarpSQL with Barman, run:
+```shell
+git clone https://github.com/Samagra-Development/WarpSQL.git
+cd WarpSQL/terraform/aws
+terraform apply -var img_warpsql=<your_warpsql_image_repository> -var img_barman=<your_barman_image_repository>
+```
+
+You can also set the password for the Postgres instance by using the `warpsql_password` variable in the Terraform script the default value is `warpsql`.
+
+This will initiate the deployment of three EC2 instances that include an Ansible controller, PostgreSQL and Barman Docker containers.These instances are provisioned on an Ubuntu Host OS and are fully configured, requiring no further setup on your end.
+
+During any subsequent launches of the WarpSQL instance, the data is recovered from the latest backup stored by Barman.
+
+To specify the size of each instance's disk, provide the desired size in gigabytes to the respective variables: `warpsql_disk_size`, `ansible_disk_size`, and `barman_disk_size` in the terraform script.
+
+
+The Barman images are based on [ubc/barman-docker](https://github.com/ubc/barman-docker). By default, Barman performs a base backup according to the cron schedule `0 4 * * *`. If you need to modify this schedule, refer to the environment variables documentation at https://github.com/ubc/barman-docker#environment-variables.
+
 ## Contribution
 
 You can contribute to the development of WarpSQL using both Gitpod and Codespaces. Follow the steps below to set up your development environment and make contributions:

barman/Dockerfile

@@ -0,0 +1,16 @@
+FROM ubcctlt/barman
+
+RUN apt-get update && \
+    apt-get install -y openssh-server && \
+    apt-get clean
+# Permit root login via SSH
+RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+RUN sed -i 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/' /etc/ssh/sshd_config
+
+# SSH port 
+EXPOSE 22
+COPY barman_entrypoint.sh  /barman_entrypoint.sh
+RUN chmod +x /barman_entrypoint.sh
+RUN mkdir -p /run/sshd
+ENTRYPOINT ["tini","--","/barman_entrypoint.sh"]
+CMD ["cron", "-L", "4",  "-f"]

barman/barman_entrypoint.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# copy ssh keys to root and barman users
+set -ex
+if [ -d "/tmp/ssh/" ]; then
+  cp -R /tmp/ssh/ /root/.ssh/
+chmod 700 /root/.ssh
+chmod 644 /root/.ssh/id_rsa.pub
+chmod 600 /root/.ssh/id_rsa
+cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys
+chmod 600 /root/.ssh/authorized_keys
+cp -R /tmp/ssh/* ~barman/.ssh/
+ls -alh ~barman/.ssh/
+ls -alh /tmp/ssh/
+ls -alh /root/.ssh/
+chown barman:barman -R  ~barman/.ssh/ 
+    su - barman -c "chmod 700 ~barman/.ssh \
+    && chmod 644 ~barman/.ssh/id_rsa.pub \
+    && chmod 600 ~barman/.ssh/id_rsa  \
+    && cat ~barman/.ssh/id_rsa.pub >> ~barman/.ssh/authorized_keys \
+    && chmod 600 ~barman/.ssh/authorized_keys"
+/usr/sbin/sshd
+fi
+exec /entrypoint.sh "$@"

terraform/aws/config/barman/barman.conf

@@ -0,0 +1,24 @@
+; Commented lines show the default values
+
+[barman]
+; archiver = off
+; backup_method = rsync
+; backup_directory = %(barman_home)s/%(name)s
+
+; This must be set to the BARMAN_DATA_DIR environment variable
+barman_home = /var/lib/barman
+
+; barman_lock_directory = %(barman_home)s
+compression = gzip
+configuration_files_direct
+ory = /etc/barman/barman.d
+;last_backup_maximum_age = 1 week
+log_file = /var/lib/barman/barman.log
+log_level = DEBUG
+;minimum_redundancy = 1
+network_compression = true
+retention_policy = RECOVERY WINDOW of 4 WEEKS
+; retention_policy_mode = auto
+reuse_backup = link
+streaming_archiver = on
+; wal_retention_policy = main

terraform/aws/config/barman/barman.d/pg.conf

@@ -0,0 +1,33 @@
+[pg]
+; active = true
+; archiver = off
+; archiver_batch_size = 0
+; backup_directory = %(barman_home)s/%(name)s
+backup_method = postgres
+; backup_options =
+; basebackup_retry_sleep = 30
+; basebackup_retry_times = 0
+; basebackups_directory = %(backup_directory)s/base
+; check_timeout = 30
+conninfo = host=pg user=barman dbname=postgres
+description = 'warpsql database'
+; disabled = false
+; errors_directory = %(backup_directory)s/errors
+; immediate_checkpoint = false
+; incoming_wals_directory = %(backup_directory)s/incoming
+; minimum_redundancy = 0
+; network_compression = false
+; path_prefix = /usr/lib/postgresql/9.5
+; recovery_options =
+; retention_policy_mode = auto
+; ssh_command = 'ssh -i /home/barman/.ssh/pg.id_rsa postgres@pg'
+slot_name = barman
+create_slot = auto
+streaming_archiver = on
+; streaming_archiver_batch_size = 0
+; streaming_archiver_name = barman_receive_wal
+; streaming_backup_name = barman_streaming_backup
+streaming_conninfo = host=pg user=streaming_barman dbname=postgres
+; streaming_wals_directory = %(backup_directory)s/streaming
+; wal_retention_policy = main
+; wals_directory = %(backup_directory)s/wals'

terraform/aws/config/init/init.sh

@@ -0,0 +1,3 @@
+# !/bin/bash
+set -e
+echo "host  replication  all   all   scram-sha-256" >> /var/lib/postgresql/data/pg_hba.conf 

terraform/aws/config/init/init.sql

@@ -0,0 +1,2 @@
+CREATE USER barman WITH SUPERUSER PASSWORD 'barman';
+CREATE USER streaming_barman WITH REPLICATION  PASSWORD 'streaming_barman';

terraform/aws/main.tf

@@ -0,0 +1,233 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 4.42.0"
+    }
+  }
+  required_version = ">= 0.14.5"
+}
+
+provider "aws" {
+  region = var.region
+}
+
+locals {
+  ami_id = "ami-053b0d53c279acc90" #ubuntu 22.04 LTS
+}
+
+resource "aws_vpc" "vpc" {
+  cidr_block           = var.cidr_vpc
+  enable_dns_support   = true
+  enable_dns_hostnames = true
+}
+
+resource "aws_internet_gateway" "igw" {
+  vpc_id = aws_vpc.vpc.id
+}
+
+resource "aws_subnet" "subnet_public" {
+  vpc_id     = aws_vpc.vpc.id
+  cidr_block = var.cidr_subnet
+}
+
+resource "aws_route_table" "rtb_public" {
+  vpc_id = aws_vpc.vpc.id
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_internet_gateway.igw.id
+  }
+}
+
+resource "aws_route_table_association" "rta_subnet_public" {
+  subnet_id      = aws_subnet.subnet_public.id
+  route_table_id = aws_route_table.rtb_public.id
+}
+
+resource "aws_security_group" "sg_22_80" {
+  name_prefix = "warpsql"
+  vpc_id      = aws_vpc.vpc.id
+  # SSH access from the VPC
+  ingress {
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+  # postgres access
+  ingress {
+    from_port   = 5432
+    to_port     = 5432
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+  ingress {
+    from_port   = 7000
+    to_port     = 7000
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "tls_private_key" "warpsql-rsa" {
+  algorithm = "RSA"
+  rsa_bits  = 4096
+}
+resource "aws_key_pair" "deployer" {
+  key_name_prefix = "warpsql"
+  public_key      = tls_private_key.warpsql-rsa.public_key_openssh
+}
+resource "local_sensitive_file" "pem_file" {
+  # write the private key to local for ssh access
+  filename        = pathexpand("./${aws_key_pair.deployer.key_name}.pem")
+  file_permission = "600"
+  content         = tls_private_key.warpsql-rsa.private_key_pem
+}
+
+resource "aws_instance" "warpsql" {
+  ami                         = local.ami_id
+  instance_type               = "t2.micro"
+  subnet_id                   = aws_subnet.subnet_public.id
+  vpc_security_group_ids      = [aws_security_group.sg_22_80.id]
+  associate_public_ip_address = true
+  key_name                    = aws_key_pair.deployer.key_name
+  root_block_device {
+    volume_size = var.warpsql_disk_size
+  }
+  user_data = <<EOF
+  #cloud-config
+    hostname: barman
+  EOF
+
+  tags = {
+    Name = "WarpSQL"
+  }
+}
+
+resource "aws_instance" "barman" {
+  ami                         = local.ami_id
+  instance_type               = "t2.micro"
+  subnet_id                   = aws_subnet.subnet_public.id
+  vpc_security_group_ids      = [aws_security_group.sg_22_80.id]
+  associate_public_ip_address = true
+  key_name                    = aws_key_pair.deployer.key_name
+  user_data                   = <<EOF
+  #cloud-config
+    hostname: barman
+  EOF
+  root_block_device {
+    volume_size = var.barman_disk_size
+  }
+  tags = {
+    Name = "Barman"
+  }
+}
+
+
+resource "aws_instance" "ansible" {
+  ami                         = local.ami_id
+  instance_type               = "t2.micro"
+  subnet_id                   = aws_subnet.subnet_public.id
+  vpc_security_group_ids      = [aws_security_group.sg_22_80.id]
+  associate_public_ip_address = true
+  key_name                    = aws_key_pair.deployer.key_name
+  user_data                   = <<EOF
+  #cloud-config
+    hostname: ansible
+  EOF
+  root_block_device {
+    volume_size = var.ansible_disk_size
+  }
+  tags = {
+    Name = "ansible"
+  }
+
+  connection {
+    type        = "ssh"
+    user        = "ubuntu"
+    private_key = tls_private_key.warpsql-rsa.private_key_pem
+    host        = self.public_ip
+  }
+  # install and setup ansible
+  provisioner "remote-exec" {
+    inline = [
+      "sudo apt update",
+      "sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install python3-pip python3.10-venv ",
+      "python3 -m pip install --user pipx",
+      "python3 -m pipx ensurepath",
+      "python3 -m pipx install --include-deps ansible",
+      "mkdir -p ~/warpsql/ssh",
+      "mkdir -p /tmp/warpsql"
+    ]
+  }
+
+
+}
+
+resource "null_resource" "warpsql" {
+  depends_on = [aws_instance.ansible, aws_instance.warpsql, aws_instance.barman]
+
+  triggers = {
+    always_run = "${timestamp()}" # always run the ansible script
+  }
+  connection {
+    type        = "ssh"
+    user        = "ubuntu"
+    private_key = tls_private_key.warpsql-rsa.private_key_pem
+    host        = aws_instance.ansible.public_ip
+  }
+
+  # save required files on ansible host
+  provisioner "file" {
+    content     = <<EOF
+barman:
+  hosts:
+    br1:
+      ansible_host: ${aws_instance.barman.private_ip}
+
+warpsql:
+  hosts:
+    wsql1:
+      ansible_host: ${aws_instance.warpsql.private_ip}
+
+EOF
+    destination = "/tmp/warpsql/inventory.yml"
+  }
+
+  provisioner "file" {
+    source      = "config"
+    destination = "/tmp/warpsql/"
+  }
+  provisioner "file" {
+    source      = local_sensitive_file.pem_file.filename
+    destination = "/tmp/warpsql/warpsql-ansible.pem"
+  }
+
+  provisioner "file" {
+    source      = "playbook-warpsql.yml"
+    destination = "/tmp/warpsql/playbook-warpsql.yml"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "cp -r /tmp/warpsql/* /home/ubuntu/warpsql",
+      "cd ~/warpsql",
+      "chmod 700 ~/warpsql/warpsql-ansible.pem",
+    "ANSIBLE_HOST_KEY_CHECKING=False /home/ubuntu/.local/bin/ansible-playbook -u ubuntu -i inventory.yml --become-method sudo --private-key 'warpsql-ansible.pem'  playbook-warpsql.yml --extra-vars 'warpsql_password=${var.warpsql_password} img_warpsql=${var.img_warpsql} img_barman=${var.img_barman}'"]
+  }
+}
+
+output "public_ip" {
+  value = {
+    WarpSQL = aws_instance.warpsql.public_ip
+    Barman  = aws_instance.barman.public_ip
+  Ansible = aws_instance.ansible.public_ip }
+}
+