👷 Automate lite integration and regression tests

.github/README/README.md

@@ -46,7 +46,7 @@ flowchart LR
       subgraph build_C-PAC.yml
         bCPAC[[C-PAC]]
       end
-      subgraph build_and_test.yml
+      subgraph build_and_test.yaml
         ubuntu[[Ubnutu]]-->stages[[stages]]-->build-base[[build-base]]-->build-base-standard[[build-base-standard]]
 
         Circle_tests[[Circle_tests]]
@@ -65,7 +65,7 @@ flowchart LR
         smoke-tests-participant[[smoke-tests-participant]]
       end
 
-      on_push.yml-->build_and_test.yml
+      on_push.yaml-->build_and_test.yaml
 
       delete_images.yml
     end
@@ -79,8 +79,8 @@ flowchart LR
 
     Circle_tests-->CircleCI((Run tests on Circle CI))
 
-    on_push.yml<-->get_pr_base_shas
-    on_push.yml-->update_all_preconfigs
+    on_push.yaml<-->get_pr_base_shas
+    on_push.yaml-->update_all_preconfigs
 
     cpacdockerfiles<-->C-PAC
 
@@ -94,7 +94,7 @@ flowchart LR
     bCPAC<-->local_ghcr
     stages<-->local_ghcr
 
-    push>git push]-->on_push.yml
+    push>git push]-->on_push.yaml
 
     smoke-tests-participant-->smoke_test_human
     smoke-tests-participant-->smoke_test_nhp

.github/workflows/build_and_test.yml → .github/workflows/build_and_test.yaml

@@ -46,6 +46,10 @@ on:
         description: 'third phase of staging images to rebuild (base images)'
         type: string
         required: true
+      test_mode:
+        description: 'lite or full?'
+        type: string
+        default: None
 
 jobs:
   Ubuntu:
@@ -186,7 +190,7 @@ jobs:
         run: |
             sed -i -e 's/^/\.github\/Dockerfiles\//' .github/stage_requirements/${{ matrix.variant }}.txt
             echo 'dev/docker_data/required_afni_pkgs.txt' >> .github/stage_requirements/${{ matrix.variant }}.txt
-            echo '.github/workflows/build_and_test.yml' >> .github/stage_requirements/${{ matrix.variant }}.txt
+            echo '.github/workflows/build_and_test.yaml' >> .github/stage_requirements/${{ matrix.variant }}.txt
             echo '.github/stage_requirements/${{ matrix.variant }}.txt' >> .github/stage_requirements/${{ matrix.variant }}.txt
       - name: Set tag & see if it exists
         continue-on-error: true
@@ -258,7 +262,7 @@ jobs:
         run: |
             sed -i -e 's/^/\.github\/Dockerfiles\//' .github/stage_requirements/standard.txt
             echo 'dev/docker_data/required_afni_pkgs.txt' >> .github/stage_requirements/standard.txt
-            echo '.github/workflows/build_and_test.yml' >> .github/stage_requirements/standard.txt
+            echo '.github/workflows/build_and_test.yaml' >> .github/stage_requirements/standard.txt
             echo '.github/stage_requirements/standard.txt' >> .github/stage_requirements/standard.txt
       - name: Set tag & see if it exists
         continue-on-error: true
@@ -327,19 +331,21 @@ jobs:
     if: github.ref_name == 'develop' || github.ref_name == 'main'
     uses: ./.github/workflows/smoke_test_participant.yml
 
-  regtest-lite:
-    name: Run lite regression test
+  check_test_mode:
+    name: check_test_mode
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo ${{ inputs.test_mode }}
+
+  regtest:
+    name: Run regression and integration test
     needs:
       - C-PAC
     secrets: inherit
-    if: contains(github.event.head_commit.message, '[run reg-suite]')
-    uses: ./.github/workflows/regression_test_lite.yml
-
-  regtest-full:
-    name: Run full regression test
-    needs:
-      - smoke-tests-participant
-    uses: ./.github/workflows/regression_test_full.yml
+    if: inputs.test_mode == 'lite'
+    uses: ./.github/workflows/regtest.yaml
+    with:
+      test_mode: ${{ inputs.test_mode }}
 
   Circle_tests:
     name: Run tests on CircleCI

.github/workflows/on_push.yml → .github/workflows/on_push.yaml

@@ -107,10 +107,36 @@ jobs:
           echo "phase_three=${phase_three}" >> $GITHUB_OUTPUT
           echo "rebuild_phase_three=${rebuild_phase_three}" >> $GITHUB_OUTPUT
 
+  check_pr:
+    runs-on: ubuntu-latest
+    outputs:
+      test_mode: ${{ steps.check_pr.outputs.test_mode }}
+    steps:
+      - name: Check out C-PAC
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+      - name: Check if commit is in a PR to develop
+        id: check_pr
+        run: |
+          TEST_MODE=none
+          if echo "${{ github.event.head_commit.message }}" | grep -q '\[run reg-suite lite\]'
+          then
+            TEST_MODE=lite
+          elif gh pr list --base develop --json number,state,draft | jq 'any(.[]; .state == "OPEN" or .draft == true)'; then
+            TEST_MODE=lite
+          elif gh pr list --base main --json number,state,draft | jq 'any(.[]; .state == "OPEN" or .draft == true)'; then
+            TEST_MODE=full
+          fi
+          echo "test_mode=${TEST_MODE}"
+          echo "test_mode=${TEST_MODE}" >> $GITHUB_OUTPUT
+
   build-stages:
     name: Build multistage image stages
-    needs: check-updated-preconfigs
-    uses: ./.github/workflows/build_and_test.yml
+    needs:
+      - check_pr
+      - check-updated-preconfigs
+    uses: ./.github/workflows/build_and_test.yaml
     secrets: inherit
     with:
       phase_one: ${{ needs.check-updated-preconfigs.outputs.phase_one }}
@@ -119,3 +145,4 @@ jobs:
       rebuild_phase_two: ${{ needs.check-updated-preconfigs.outputs.rebuild_phase_two }}
       phase_three: ${{ needs.check-updated-preconfigs.outputs.phase_three }}
       rebuild_phase_three: ${{ needs.check-updated-preconfigs.outputs.rebuild_phase_three }}
+      test_mode: ${{ needs.check_pr.outputs.test_mode }}

.github/workflows/regtest.yaml

@@ -0,0 +1,115 @@
+name: Launch  regression test
+
+on:
+  pull_request:
+    branches:
+      - develop
+    types:
+      - opened
+      - ready_for_review
+      - reopened
+  workflow_call:
+    inputs:
+      test_mode:
+        type: string
+        required: true
+    secrets:
+      GH_CLI_BIN_PATH:
+        description: 'path to directory containing GitHub CLI binary if not on default $PATH'
+        required: false
+      SSH_PRIVATE_KEY:
+        required: true
+      SSH_USER:
+        required: true
+      SSH_HOST:
+        required: true
+      SSH_WORK_DIR:
+        required: true
+  workflow_dispatch:
+    inputs:
+      test_mode:
+        type: string
+        required: true
+
+jobs:
+  test:
+    name: Regression Test - ${{ inputs.test_mode }}
+    environment: ACCESS
+    env:
+      COMPARISON_PATH: ${{ secrets.COMPARISON_PATH }}
+      DASHBOARD_REPO: ${{ vars.DASHBOARD_REPO}}
+      DOCKER_TAG:
+      GH_CLI_BIN_PATH: ${{ secrets.GH_CLI_BIN_PATH }}
+      SLURM_TESTING_BRANCH: ${{ vars.SLURM_TESTING_BRANCH }}
+      SLURM_TESTING_PACKAGE: ${{ vars.SLURM_TESTING_PACKAGE }}
+      SLURM_TESTING_REPO: ${{ vars.SLURM_TESTING_REPO }}
+      SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
+      SSH_HOST: ${{ secrets.SSH_HOST }}
+      SSH_USER: ${{ secrets.SSH_USER }}
+      SSH_WORK_DIR: ${{ secrets.SSH_WORK_DIR }}
+      TOKEN_FILE: ${{ secrets.TOKEN_FILE }}
+    if: |
+      ${{ github.env.SSH_PRIVATE_KEY != '' &&
+      (github.event_name == 'workflow_dispatch' ||
+       (github.event_name == 'pull_request' &&
+        github.event.repository.fork == 'false')) }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get C-PAC branch
+        run: |
+          if [[ ! $GITHUB_REF_NAME == 'main' ]] && [[ ! $GITHUB_REF_NAME == 'develop' ]]
+          then
+            TAG=${GITHUB_REF_NAME//\//_}
+          elif [[ $GITHUB_REF_NAME == 'develop' ]]
+          then
+            TAG=nightly
+          elif [[ $GITHUB_REF_NAME == 'main' ]]
+          then
+            TAG=latest
+          fi
+          TAG=$TAG$VARIANT
+          echo DOCKER_TAG=$(echo "ghcr.io/${{ github.repository }}" | tr '[:upper:]' '[:lower:]'):$TAG >> $GITHUB_ENV
+          cat $GITHUB_ENV
+
+      - name: Install SSH Keys
+        run: |
+          mkdir -p ~/.ssh/
+          echo "${{ env.SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa
+          chmod 600 ~/.ssh/id_rsa
+          ssh-keyscan -H -t rsa "${{ env.SSH_HOST }}" > ~/.ssh/known_hosts
+
+      - name: Connect and Run Regression Test ${{ inputs.test_mode }}
+        uses: appleboy/[email protected]
+        with:
+          host: ${{ env.SSH_HOST }}
+          username: ${{ env.SSH_USER }}
+          key: ${{ env.SSH_PRIVATE_KEY }}
+          command_timeout: 200m
+          script: |
+            set -x
+            cd ${{ env.SSH_WORK_DIR }}
+            if pip show "${{ env.SLURM_TESTING_PACKAGE }}" > /dev/null 2>&1; then
+                # If the package is installed, upgrade it
+                python3 -m pip install --user --upgrade --force-reinstall "https://github.com/${{ env.SLURM_TESTING_REPO }}/archive/${{ env.SLURM_TESTING_BRANCH }}.zip"
+            else
+                # If the package is not installed, install it
+                python3 -m pip install --user "https://github.com/${{ env.SLURM_TESTING_REPO }}/archive/${{ env.SLURM_TESTING_BRANCH }}.zip"
+            fi
+            mkdir -p ./logs/${{ github.sha }}
+            sbatch cpac-slurm-status launch \
+              --wd="${{ env.SSH_WORK_DIR }}/logs/${{ github.sha }}" \
+              --comparison-path="${{ env.COMPARISON_PATH }}" \
+              --dashboard-repo="${{ env.DASHBOARD_REPO}}" \
+              --home-dir="${{ env.SSH_WORK_DIR }}" \
+              --image="${{ env.DOCKER_TAG }}" \
+              --owner="${{ github.repository_owner }}" \
+              --path-extra="${{ env.GH_CLI_BIN_PATH }}" \
+              --repo="${{ github.repository }}" \
+              --sha="${{ github.sha }}" \
+              --slurm-testing-branch="${{ env.SLURM_TESTING_BRANCH }}" \
+              --slurm-testing-repo="${{ env.SLURM_TESTING_REPO }}" \
+              --token-file="${{ env.TOKEN_FILE }}"
+
+      - name: Cleanup SSH
+        run: |
+          rm -rf ~/.ssh

CHANGELOG.md

@@ -30,6 +30,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Moved `pygraphviz` from requirements to `graphviz` optional dependencies group.
 - Automatically tag untagged `subject_id` and `unique_id` as `!!str` when loading data config files.
 - Made orientation configurable (was hard-coded as "RPI").
+- Updated GitHub Actions to run automated integration and regression tests on HPC.
 
 ### Fixed