Merge branch 'main' into maintenance-css/T343644

.gitattributes

@@ -0,0 +1,6 @@
+# Directory entries are not enough to encrypt fines beneath it
+# https://github.com/AGWA/git-crypt#gitattributes-file
+quarry/config-prod.yaml filter=git-crypt diff=git-crypt
+tofu/secrets.tf filter=git-crypt diff=git-crypt
+helm-quarry/prod-config.yaml filter=git-crypt diff=git-crypt
+secrets.sh filter=git-crypt diff=git-crypt

.github/workflows/build-and-push.yaml

@@ -0,0 +1,40 @@
+---
+
+name: Build and Push quarry image
+
+'on':
+  pull_request_target:
+
+jobs:
+  build_and_push:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: git fetch
+        run: |
+          git fetch
+      - name: git checkout
+        run: |
+          git checkout ${{ github.head_ref }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_QUARRY_USER }}
+          password: ${{ secrets.QUAY_QUARRY_PASSWORD }}
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          context: .
+          tags: quay.io/wikimedia-quarry/quarry:pr-${{ github.event.number }}
+
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}

.github/workflows/linter.yaml

@@ -0,0 +1,19 @@
+---
+
+name: "linter"
+
+'on':
+  pull_request:
+
+jobs:
+  tox:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: build tox container
+        run: |
+          docker build -f tests/tox/Dockerfile -t tox:01 .
+      - name: run tox container
+        run: |
+          docker run tox:01

.github/workflows/update-container-tags.yaml

@@ -0,0 +1,45 @@
+---
+
+name: update container tags
+
+'on':
+  pull_request_target:
+
+jobs:
+  update-container-tags:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+
+      - name: git fetch
+        run: |
+          git fetch
+      - name: git checkout
+        run: |
+          git checkout ${{ github.head_ref }}
+
+      # compare  to upstream main branch
+      - name: git add remote
+        run: |
+          git remote add quarry https://github.com/toolforge/quarry.git
+      - name: git remote update
+        run: |
+          git remote update
+
+      - name: update values.yaml
+        run: |
+          for i in web worker ; do
+            sed -i "s/tag: .* # ${i} tag managed by github actions$/tag: pr-${{ github.event.number }} # ${i} tag managed by github actions/" helm-quarry/values.yaml
+          done
+
+      - uses: EndBug/add-and-commit@v7
+        with:
+          add: 'helm-quarry/values.yaml'
+          author_name: Github Action
+          author_email: [email protected]
+          branch: ${{ github.head_ref }}
+          message: 'auto update of ${{ inputs.imagename }} tag'
+          pull: --rebase --autostash

.gitignore

@@ -14,3 +14,9 @@ config.yaml
 .python-version
 .tox/
 *.swp
+.vscode
+.tool-versions
+terraform.tfstate*
+.terraform*
+tofu/kube.config
+ansible/collections/*

Dockerfile

@@ -1,30 +1,24 @@
 # Use official python base image, small and debian edition
-FROM amd64/python:3.7.3-slim
-
-ARG purpose=dev
-
-# Update debian packages
-RUN apt-get update && \
-    apt-get upgrade -y
+FROM amd64/python:3.7.16-slim
 
 # Create Quarry user, create /results folder owned by this user,
-# to be mounted as volume to be shared between web and runner
+# to be mounted as volume to be shared between web and runner in dev setup
 RUN useradd -r -m quarry && \
     mkdir /results && \
     chown -R quarry: /results
 
 WORKDIR /app
 
 COPY requirements.txt /app
-# Install python or test dependencies
-RUN if [ ${purpose} = "test" ] ; then apt-get install -y tox redis-server; \
-    else pip install --upgrade pip wheel && \
-    pip install -r requirements.txt; fi
+# Install dependencies
+RUN pip install --upgrade pip wheel && \
+    pip install -r requirements.txt
 
 # Copy app code
 USER quarry
 COPY . /app
 
-# Run web server
+# Expose port for web server
 EXPOSE 5000
-ENTRYPOINT ["python", "quarry.wsgi"]
+
+# Entrypoint is set elsewhere, as it's different for web and worker

README.md

@@ -4,6 +4,7 @@ queries against Wikipedia and sister projects databases.
 
 ## Setting up a local dev environment ##
 
+# docker-compose
 Quarry uses [Docker](https://docs.docker.com/engine/install/) to set up a local
 environment. You can set it up by:
 
@@ -18,43 +19,50 @@ will imediatelly be taken into account.
 A worker node is also created to execute your queries in the background (uses the
 same image). Finally, redis and two database instances are also started.
 
-One database is your quarry database the other is a wikireplica-like database
-named `mywiki`. This (or `mywiki_p`) is the correct thing to enter in the
-database field on all local test queries.
-
-In your local environment, you can query Quarry internal db itself. Use then
-"quarry" as database name.
-
 To stop, run `docker-compose stop` or hit CTRL-C on the terminal your docker-compose
 is running in. After that, to start with code changes, you'll want to `docker-compose down`
 to clean up. Also, this creates a docker volume where sqlite versions of query
 results are found. That will not be cleaned up unless you run `docker-compose down -v`
 
 
+
+# minikube
+It is possible to run a quarry system inside [minikube](https://minikube.sigs.k8s.io/docs/)!
+At this time, you need to set it up with a cluster version before 1.22, most likely.
+
+First build the containers:
+```
+eval $(minikube docker-env)
+docker build . -t quarry:01
+cd docker-replica/
+docker build . -t mywiki:01
+```
+
+You will need to install minikube (tested on minikube 1.23) and [helm](https://helm.sh) and kubectl on your system. When you are confident those are working, start minikube with:
+ - `minikube start --kubernetes-version=v1.23.15`
+ - `minikube addons enable ingress`
+ - `kubectl create namespace quarry`
+ - `helm -n quarry install quarry helm-quarry -f helm-quarry/dev-env.yaml`
+
+The rest of the setup instructions will display on screen as long as the install is successful.
+
+# local databases
+Both local setups will create two databases.
+
+One database is your quarry database the other is a wikireplica-like database
+named `mywiki`. This (or `mywiki_p`) is the correct thing to enter in the
+database field on all local test queries.
+
+The other database is the Quarry internal db. In your local environment, you can query Quarry internal db itself. Use then
+"quarry" as database name.
+
 ### Updating existing containers ###
 
 If you had already run a dev environment (that is, ran `docker-compose up`) you might want to update
 the containers with the new dependencies by running `docker-compose build` before running
 `docker-compose up` again.
 
 
-## Running tests ##
-
-1. Set up [Blubber](https://wikitech.wikimedia.org/wiki/Blubber) to run tests:
-https://wikitech.wikimedia.org/wiki/Blubber/Download
-```bash
-blubber() {
-  if [ $# -lt 2 ]; then
-    echo 'Usage: blubber config.yaml variant'
-    return 1
-  fi
-  curl -s -H 'content-type: application/yaml' --data-binary @"$1" https://blubberoid.wikimedia.org/v1/"$2"
-}
-```
-2. Run tests:
-`blubber .pipeline/blubber.yaml quarry-test | docker build --tag blubber-quarry:01 --file - . ; docker run blubber-quarry:01`
-
-
 ## Useful commands ##
 
 To pre-compile nunjucks templates:
@@ -71,3 +79,31 @@ Bug: <ticket number>
 
 For example:
 Bug: T317566
+
+## git-crypt ##
+
+git-crypt is used to encrypt the config.yaml file. To decrypt ask a maintainer for the decryption key and:
+```
+git clone https://github.com/toolforge/quarry.git
+cd quarry
+git-crypt unlock <path to decryption key>
+```
+
+## Deploying to production ##
+From the quarry-bastion:
+`git clone https://github.com/toolforge/quarry.git`
+`cd quarry`
+`git checkout <branch>` If not deploying main
+`git-crypt unlock <path to key>`
+`bash deploy.sh`
+In horizon point the web proxy at the new cluster.
+
+### Fresh deploy ###
+For a completely fresh deploy, and nfs server will need to be setup. Add its hostname to helm-quarry/prod-env.yaml.
+And an object store will need to be generated for the tofu state file. Named "tofu-state"
+And setup mysql:
+`mysql -uquarry -h <trove hostname created in by tofu> -p < schema.sql`
+
+## troubleshooting ##
+If ansible doesn't detect a change for quarry helm the following can be run:
+`helm -n quarry upgrade --install quarry helm-quarry -f helm-quarry/prod-env.yaml`

ansible/ansible.cfg

@@ -0,0 +1,13 @@
+[defaults]
+# Better error output
+stdout_callback=debug
+stderr_callback=debug
+
+collections_path=./collections/ansible_collections
+
+# we're only using localhost, no need for the warning.
+localhost_warning=False
+
+[inventory]
+# Only using localhost, so no inventory
+inventory_unparsed_warning=False

ansible/quarry.yaml

@@ -0,0 +1,14 @@
+---
+
+- name: Deploy quarry to k8s cluster
+  gather_facts: false
+  hosts: localhost
+  tasks:
+    - name: Deploy quarry
+      kubernetes.core.helm:
+        name: quarry
+        chart_ref: ../helm-quarry
+        release_namespace: "quarry"
+        create_namespace: true
+        values_files:
+          - ../helm-quarry/prod-env.yaml

deploy.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -e
+
+if ! command -v kubectl ; then
+  echo "please install kubectl"
+  exit 1
+fi
+
+if ! command -v helm ; then
+  echo "please install helm"
+  exit 1
+fi
+
+if ! command -v tofu; then
+  echo "please install tofu"
+  exit 1
+fi
+
+source secrets.sh
+
+python3 -m venv .venv/deploy
+source .venv/deploy/bin/activate
+pip install ansible==8.1.0 kubernetes==26.1.0
+# install helm diff. Needed to keep helm module idempotent
+helm plugin install https://github.com/databus23/helm-diff || true
+
+cd tofu
+AWS_ACCESS_KEY_ID=${ACCESS_KEY} AWS_SECRET_ACCESS_KEY=${SECRET_KEY} tofu init
+AWS_ACCESS_KEY_ID=${ACCESS_KEY} AWS_SECRET_ACCESS_KEY=${SECRET_KEY} tofu apply # -auto-approve
+export KUBECONFIG=$(pwd)/kube.config
+
+cd ../ansible
+# install collections here to take advantage of ansible.cfg configs
+ansible-galaxy collection install -U kubernetes.core -p ./collections
+
+ansible-playbook quarry.yaml
+#kubectl create namespace quarry --dry-run=client -o yaml | kubectl apply -f -
+#helm -n quarry upgrade --install quarry helm-quarry -f helm-quarry/prod-env.yaml
+

docker-compose.yml

@@ -4,6 +4,9 @@ services:
 
   web:
     build: .
+    # To mimic a production runtime, uncomment:
+    # entrypoint: ["gunicorn", "-w", "2", "--bind", "0.0.0.0:5000", "wsgi:application"]
+    entrypoint: ["python", "wsgi.py"]
     volumes:
       - .:/app
       - results:/results
@@ -20,12 +23,14 @@ services:
       - .:/app
       - results:/results
     entrypoint: ["celery", "--app", "quarry.web.worker", "worker", ]
+    extra_hosts:
+    - "host.docker.internal:host-gateway"
     depends_on:
       - "db"
       - "redis"
 
   redis:
-    image: redis:alpine
+    image: redis:alpine3.18
 
   # wikireplica simulator
   mywiki:

docker-replica/Dockerfile

@@ -0,0 +1,3 @@
+FROM mariadb:10.4-focal
+
+COPY replica.sql /docker-entrypoint-initdb.d/replica.sql