quarry on k8s

.gitattributes

@@ -1,3 +1,5 @@
 # Directory entries are not enough to encrypt fines beneath it
 # https://github.com/AGWA/git-crypt#gitattributes-file
 quarry/config-prod.yaml filter=git-crypt diff=git-crypt
+tofu/secrets.tf filter=git-crypt diff=git-crypt
+helm-quarry/prod-config.yaml filter=git-crypt diff=git-crypt

.gitignore

@@ -16,3 +16,6 @@ config.yaml
 *.swp
 .vscode
 .tool-versions
+terraform.tfstate*
+.terraform*
+tofu/kube.config

README.md

@@ -105,3 +105,9 @@ git clone https://github.com/toolforge/quarry.git
 cd quarry
 git-crypt unlock <path to decryption key>
 ```
+
+## Deploying to production ##
+From the quarry-bastion in the git checkout that has the state file.
+`bash deploy.sh`
+`mysql -uquarry -h <trove hostname created in last step> -p < schema.sql`
+In horizon point the web proxy at the new cluster.

ansible/ansible.cfg

@@ -0,0 +1,5 @@
+[defaults]
+
+# Better error output
+stdout_callback=debug
+stderr_callback=debug

ansible/quarry.yaml

@@ -0,0 +1,14 @@
+---
+
+- name: Deploy quarry to k8s cluster
+  gather_facts: false
+  hosts: localhost
+  tasks:
+    - name: Deploy quarry
+      kubernetes.core.helm:
+        name: quarry
+        chart_ref: ../helm-quarry
+        release_namespace: "quarry"
+        create_namespace: true
+        values_files:
+          - ../helm-quarry/prod-env.yaml

deploy.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -e
+
+if ! command -v kubectl ; then
+  echo "please install kubectl"
+  exit 1
+fi
+
+if ! command -v helm ; then
+  echo "please install helm"
+  exit 1
+fi
+
+if ! command -v tofu; then
+  echo "please install tofu"
+  exit 1
+fi
+
+python3 -m venv .venv/deploy
+source .venv/deploy/bin/activate
+pip install ansible==8.1.0 kubernetes==26.1.0
+
+cd tofu
+tofu init
+tofu apply # -auto-approve
+export KUBECONFIG=$(pwd)/kube.config
+
+cd ../ansible
+ansible-playbook quarry.yaml
+#kubectl create namespace quarry --dry-run=client -o yaml | kubectl apply -f -
+#helm -n quarry upgrade --install quarry helm-quarry -f helm-quarry/prod-env.yaml
+

helm-quarry/prod-env.yaml

@@ -9,3 +9,6 @@ worker:
   replicas: 2
   memory: "700Mi"
   cpu: "100m"
+
+nfs:
+  server: k8s-nfs.quarry.eqiad1.wikimedia.cloud

helm-quarry/templates/config_yaml.yaml

@@ -4,7 +4,7 @@ metadata:
   name: config
 data:
 {{ if .Values.localdev.enabled }}
-  config.yaml: {{ .Files.Get "dev_config.yaml" | quote }}
+  config.yaml: {{ .Files.Get "dev-config.yaml" | quote }}
 {{ else }}
-  config.yaml: {{ .Files.Get "prod_config.yaml" | quote }}
+  config.yaml: {{ .Files.Get "prod-config.yaml" | quote }}
 {{ end }}

helm-quarry/templates/db_deployment.yaml

@@ -1,3 +1,4 @@
+{{ if .Values.localdev.enabled }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -34,3 +35,4 @@ spec:
         - name: schema
           configMap:
             name: db-schema
+{{ end }}

helm-quarry/templates/db_schema_cm.yaml

@@ -1,7 +1,9 @@
+{{ if .Values.localdev.enabled }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: db-schema
 data:
   schema.sql: |-
 {{ .Files.Get "schema.sql" | indent 4}}
+{{ end }}

helm-quarry/templates/db_service.yaml

@@ -1,3 +1,4 @@
+{{ if .Values.localdev.enabled }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -9,3 +10,4 @@ spec:
   - name: db
     port: 3306
     targetPort: 3306
+{{ end }}

helm-quarry/templates/mywiki_deployment.yaml

@@ -1,3 +1,4 @@
+{{ if .Values.localdev.enabled }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -27,3 +28,4 @@ spec:
             value: "repl"
           - name: MYSQL_RANDOM_ROOT_PASSWORD
             value: "1"
+{{ end }}

helm-quarry/templates/mywiki_service.yaml

@@ -1,3 +1,4 @@
+{{ if .Values.localdev.enabled }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -9,3 +10,4 @@ spec:
   - name: mywiki
     port: 3306
     targetPort: 3306
+{{ end }}

helm-quarry/templates/results_pv.yaml

@@ -1,3 +1,4 @@
+{{ if .Values.localdev.enabled }}
 apiVersion: v1
 kind: PersistentVolume
 metadata:
@@ -10,3 +11,4 @@ spec:
     - ReadWriteMany
   hostPath:
     path: "/results"
+{{ end }}

helm-quarry/templates/results_pvc.yaml

@@ -1,3 +1,4 @@
+{{ if .Values.localdev.enabled }}
 kind: PersistentVolumeClaim
 apiVersion: v1
 metadata:
@@ -7,3 +8,4 @@ spec:
   resources:
     requests:
       storage: 1Gi
+{{ end }}

helm-quarry/templates/web_deployment.yaml

@@ -20,6 +20,19 @@ spec:
           imagePullPolicy: Always
           command: ["gunicorn"]
           args: ["-w", "2", "--bind", "0.0.0.0:5000", "wsgi:application"]
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 5000
+            initialDelaySeconds: 15
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /
+              port: 5000
+            initialDelaySeconds: 90
+            periodSeconds: 30
+            failureThreshold: 4
           volumeMounts:
             - mountPath: "/results"
               name: results
@@ -32,8 +45,14 @@ spec:
               cpu: {{ .Values.web.cpu }}
       volumes:
         - name: results
+{{ if .Values.localdev.enabled }}
           persistentVolumeClaim:
             claimName: results
+{{ else }}
+          nfs:
+            server: {{ .Values.nfs.server }}
+            path: /srv/quarry/project/quarry/results/
+{{ end }}
         - name: config
           configMap:
             name: config

helm-quarry/templates/web_ingress.yaml

@@ -1,3 +1,4 @@
+{{ if .Values.localdev.enabled }}
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -18,3 +19,4 @@ spec:
               number: 5000
         path: /
         pathType: Prefix
+{{ end }}

helm-quarry/templates/worker_deployment.yaml

@@ -32,8 +32,14 @@ spec:
               cpu: {{ .Values.worker.cpu }}
       volumes:
         - name: results
+{{ if .Values.localdev.enabled }}
           persistentVolumeClaim:
             claimName: results
+{{ else }}
+          nfs:
+            server: {{ .Values.nfs.server }}
+            path: /srv/quarry/project/quarry/results/
+{{ end }}
         - name: config
           configMap:
             name: config

tofu/123.tf

@@ -0,0 +1,33 @@
+resource "openstack_containerinfra_cluster_v1" "k8s_123_2" {
+  name                = "quarry-123-2"
+  cluster_template_id = resource.openstack_containerinfra_clustertemplate_v1.template_123_2.id
+  master_count        = 1
+  node_count          = 2
+}
+
+resource "local_file" "kube_config" {
+  content  = resource.openstack_containerinfra_cluster_v1.k8s_123_2.kubeconfig.raw_config
+  filename = "kube.config"
+}
+
+resource "openstack_containerinfra_clustertemplate_v1" "template_123_2" {
+  name                  = "quarry-123-2"
+  coe                   = "kubernetes"
+  dns_nameserver        = "8.8.8.8"
+  docker_storage_driver = "overlay2"
+  docker_volume_size    = 20
+  external_network_id   = "wan-transport-eqiad"
+  fixed_subnet          = "cloud-instances2-b-eqiad"
+  fixed_network         = "lan-flat-cloudinstances2b"
+  flavor                = "g3.cores4.ram8.disk20"
+  floating_ip_enabled   = "false"
+  image                 = "Fedora-CoreOS-38"
+  master_flavor         = "g3.cores2.ram4.disk20"
+  network_driver        = "flannel"
+
+  labels = {
+    kube_tag               = "v1.23.15-rancher1-linux-amd64"
+    hyperkube_prefix       = "docker.io/rancher/"
+    cloud_provider_enabled = "true"
+  }
+}

tofu/db.tf

@@ -0,0 +1,26 @@
+resource "openstack_db_instance_v1" "mariadb" {
+  region    = "eqiad1-r"
+  name      = "quarry-k8s"
+  flavor_id = "bb8bee7e-d8f9-460b-8344-74f745c139b9"
+  size      = 10
+
+  network {
+    uuid = "7425e328-560c-4f00-8e99-706f3fb90bb4"
+  }
+
+  user {
+    name      = "quarry"
+    host      = "%"
+    password  = var.db_password
+    databases = ["quarry"]
+  }
+
+  database {
+    name = "quarry"
+  }
+
+  datastore {
+    version = "10.5.10"
+    type    = "mariadb"
+  }
+}

tofu/main.tf

@@ -0,0 +1,17 @@
+terraform {
+  # license is incompatable at version 1.6.0
+  required_version = "= 1.6.2"
+  required_providers {
+    openstack = {
+      source  = "terraform-provider-openstack/openstack"
+      version = "~> 1.51.0"
+    }
+  }
+}
+
+provider "openstack" {
+  auth_url                      = var.auth-url
+  tenant_id                     = var.tenant_id
+  application_credential_id     = var.application_credential_id
+  application_credential_secret = var.application_credential_secret
+}

tofu/vars.tf

@@ -0,0 +1,13 @@
+# connection vars
+variable "auth-url" {
+  type = string
+  default = "https://openstack.eqiad1.wikimediacloud.org:25000"
+}
+variable "tenant_id" {
+  type = string
+  default = "quarry"
+}
+variable "application_credential_id" {
+  type = string
+  default = "4917ce71b98e498e8a6c5814b095b28e"
+}