add Disaster Recovery for AWS

Signed-off-by: Piyush Raj <[email protected]>

Dockerfile

@@ -341,3 +341,39 @@ RUN set -eux \
     && rm -rf /tmp/postgresql-hll-${POSTGRES_HLL_VERSION} /tmp/postgresql-hll-${POSTGRES_HLL_VERSION}.zip \
     && apk del .postgresql-hll-build-deps 
 
+
+# install barman client
+RUN apk add --no-cache \
+    gcc \
+    python3 \
+    rsync \
+    py3-pip \
+    python3-dev \
+    git \
+    openssh \
+    musl-dev \
+    && cd /tmp \
+    && git clone https://github.com/EnterpriseDB/barman \
+    && cd barman \
+    && ./setup.py install \
+    && ./setup.py build \
+    cd / \
+    && rm -rf /tmp/barman 
+
+# Install SSH server
+RUN apk add --no-cache openssh-server 
+
+
+# Permit root login via SSH
+RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+
+# Unloack the postgres account for ssh
+RUN passwd -u postgres
+
+# SSH port 
+EXPOSE 22
+
+COPY entrypoint.sh  /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+ENTRYPOINT [ "/entrypoint.sh" ]
+CMD ["postgres"]

README.md

@@ -96,6 +96,50 @@ packer build -var="image_repository=your_value" -var="image_tags=[tag1,tag2]" wa
 |[PgRepack](https://github.com/reorg/pg_repack)        | `pg_repack`      |
 |[PG Auto Failover](https://github.com/hapostgres/pg_auto_failover)| `pgautofailover` |
 |[HyperLogLog](https://github.com/citusdata/postgresql-hll)     | `hll`            |
+
+## Disaster recovery 
+WarpSQL includes [`barman`](https://github.com/EnterpriseDB/barman) as the disaster recovery solution
+supported platform : `aws`
+### Docker
+To launch WarpSQL with Barman, run:
+```shell
+cd terraform/docker
+terraform apply
+```
+
+There are two modules available: 
+- `module.warpsql-containers` 
+- `module.warpsql-volumes` contains the docker volumes for persistent data storage of PostgreSQL and Barman.
+
+to only destroy the containers run you can specify the target  
+```shell
+terraform destroy -target module.warpsql-containers
+```
+
+### AWS 
+WarpSQL provides a streamlined approach to deploying and managing PostgreSQL databases on AWS EC2 instances, complete with a disaster recovery solution powered by Barman. 
+> **Warning**
+WarpSQL is a work in progress, and the current setup allows public SSH access to instances, which might not be secure.
+
+To get started, ensure you have your AWS credentials set up and Terraform installed.
+
+To launch WarpSQL with Barman, run:
+```shell
+git clone https://github.com/Samagra-Development/WarpSQL.git
+cd WarpSQL/terraform/aws
+terraform apply
+```
+
+This will initiate the deployment of three EC2 instances that include an Ansible controller, PostgreSQL and Barman Docker containers.These instances are provisioned on an Ubuntu Host OS and are fully configured, requiring no further setup on your end.
+
+During any subsequent launches of the WarpSQL instance, the data is recovered from the latest backup stored by Barman.
+
+To specify the size of each instance's disk, provide the desired size in gigabytes to the respective variables: `warpsql_disk_size`, `ansible_disk_size`, and `barman_disk_size` in the terraform script.
+
+You can also set the password for the Postgres instance by using the `warpsql_password` variable in the Terraform script.
+
+The Barman images are based on [ubc/barman-docker](https://github.com/ubc/barman-docker). By default, Barman performs a base backup according to the cron schedule `0 4 * * *`. If you need to modify this schedule, refer to the environment variables documentation at https://github.com/ubc/barman-docker#environment-variables.
+
 ## Contribution
 
 You can contribute to the development of WarpSQL using both Gitpod and Codespaces. Follow the steps below to set up your development environment and make contributions:

barman/Dockerfile

@@ -0,0 +1,16 @@
+FROM ubcctlt/barman
+
+RUN apt-get update && \
+    apt-get install -y openssh-server && \
+    apt-get clean
+# Permit root login via SSH
+RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+RUN sed -i 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/' /etc/ssh/sshd_config
+
+# SSH port 
+EXPOSE 22
+COPY barman_entrypoint.sh  /barman_entrypoint.sh
+RUN chmod +x /barman_entrypoint.sh
+RUN mkdir -p /run/sshd
+ENTRYPOINT ["tini","--","/barman_entrypoint.sh"]
+CMD ["cron", "-L", "4",  "-f"]

barman/barman_entrypoint.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# copy ssh keys to root and barman users
+set -ex
+if [ -d "/tmp/ssh/" ]; then
+  cp -R /tmp/ssh/ /root/.ssh/
+chmod 700 /root/.ssh
+chmod 644 /root/.ssh/id_rsa.pub
+chmod 600 /root/.ssh/id_rsa
+cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys
+chmod 600 /root/.ssh/authorized_keys
+cp -R /tmp/ssh/* ~barman/.ssh/
+ls -alh ~barman/.ssh/
+ls -alh /tmp/ssh/
+ls -alh /root/.ssh/
+chown barman:barman -R  ~barman/.ssh/ 
+    su - barman -c "chmod 700 ~barman/.ssh \
+    && chmod 644 ~barman/.ssh/id_rsa.pub \
+    && chmod 600 ~barman/.ssh/id_rsa  \
+    && cat ~barman/.ssh/id_rsa.pub >> ~barman/.ssh/authorized_keys \
+    && chmod 600 ~barman/.ssh/authorized_keys"
+/usr/sbin/sshd
+fi
+exec /entrypoint.sh "$@"

entrypoint.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# copy ssh keys to root and postgres users
+set -e
+if [ -d "/tmp/ssh/" ]; then
+    cp -R /tmp/ssh/ /root/.ssh/
+    chmod 700 /root/.ssh
+    chmod 644 /root/.ssh/id_rsa.pub
+    chmod 600 /root/.ssh/id_rsa
+    cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys
+    chmod 600 /root/.ssh/authorized_keys
+    cp -R /tmp/ssh/ ~postgres/.ssh/ 
+    echo -e  > ~postgres/.ssh/config "Host *\n\tStrictHostKeyChecking no" # prevent barman commands failing when using ssh
+    chown postgres:postgres -R  ~postgres/.ssh/ 
+    su - postgres -c "chmod 700 ~postgres/.ssh \
+    && chmod 644 ~postgres/.ssh/id_rsa.pub \
+    && chmod 600 ~postgres/.ssh/id_rsa  \
+    && chmod 600 ~postgres/.ssh/config  \
+    && cat ~postgres/.ssh/id_rsa.pub >> ~postgres/.ssh/authorized_keys \
+    && chmod 600 ~postgres/.ssh/authorized_keys"
+/usr/sbin/sshd #start the ssh server
+fi
+
+exec /usr/local/bin/docker-entrypoint.sh "$@"

terraform/aws/config/barman/barman.conf

@@ -0,0 +1,24 @@
+; Commented lines show the default values
+
+[barman]
+; archiver = off
+; backup_method = rsync
+; backup_directory = %(barman_home)s/%(name)s
+
+; This must be set to the BARMAN_DATA_DIR environment variable
+barman_home = /var/lib/barman
+
+; barman_lock_directory = %(barman_home)s
+compression = gzip
+configuration_files_direct
+ory = /etc/barman/barman.d
+;last_backup_maximum_age = 1 week
+log_file = /var/lib/barman/barman.log
+log_level = DEBUG
+;minimum_redundancy = 1
+network_compression = true
+retention_policy = RECOVERY WINDOW of 4 WEEKS
+; retention_policy_mode = auto
+reuse_backup = link
+streaming_archiver = on
+; wal_retention_policy = main

terraform/aws/config/barman/barman.d/pg.conf

@@ -0,0 +1,33 @@
+[pg]
+; active = true
+; archiver = off
+; archiver_batch_size = 0
+; backup_directory = %(barman_home)s/%(name)s
+backup_method = postgres
+; backup_options =
+; basebackup_retry_sleep = 30
+; basebackup_retry_times = 0
+; basebackups_directory = %(backup_directory)s/base
+; check_timeout = 30
+conninfo = host=pg user=barman dbname=postgres
+description = 'warpsql database'
+; disabled = false
+; errors_directory = %(backup_directory)s/errors
+; immediate_checkpoint = false
+; incoming_wals_directory = %(backup_directory)s/incoming
+; minimum_redundancy = 0
+; network_compression = false
+; path_prefix = /usr/lib/postgresql/9.5
+; recovery_options =
+; retention_policy_mode = auto
+; ssh_command = 'ssh -i /home/barman/.ssh/pg.id_rsa postgres@pg'
+slot_name = barman
+create_slot = auto
+streaming_archiver = on
+; streaming_archiver_batch_size = 0
+; streaming_archiver_name = barman_receive_wal
+; streaming_backup_name = barman_streaming_backup
+streaming_conninfo = host=pg user=streaming_barman dbname=postgres
+; streaming_wals_directory = %(backup_directory)s/streaming
+; wal_retention_policy = main
+; wals_directory = %(backup_directory)s/wals'

terraform/aws/config/init/init.sh

@@ -0,0 +1,3 @@
+# !/bin/bash
+set -e
+echo "host  replication  all   all   scram-sha-256" >> /var/lib/postgresql/data/pg_hba.conf 

terraform/aws/config/init/init.sql

@@ -0,0 +1,2 @@
+CREATE USER barman WITH SUPERUSER PASSWORD 'barman';
+CREATE USER streaming_barman WITH REPLICATION  PASSWORD 'streaming_barman';