Merge branch 'feature/backend_separation' into 'master'

CAiMIRA API development See merge request caimira/caimira!502
CERN · Sep 13, 2024 · 048c791 · 048c791
2 parents 6f77b6e + 2998df0
commit 048c791
Show file tree

Hide file tree

Showing 188 changed files with 1,647 additions and 1,335 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -24,6 +24,7 @@ jobs:
     env:
       PROJECT_ROOT: ./
       PROJECT_NAME: caimira
+      CAIMIRA_TESTS_CALCULATOR_TIMEOUT: 30
     steps:
       - name: Checkout
         uses: actions/checkout@v2

diff --git a/.gitignore b/.gitignore
@@ -3,6 +3,7 @@ __pycache__
 *.egg-info
 *.DS_Store
 *.pyc
+dist
 
 # Editor stuff
 *.swp
@@ -18,4 +19,4 @@ app-config/openshift/caimira-test
 app-config/openshift/caimira-prod
 
 # documentation build folder
-caimira/docs/_build
+caimira/docs/_build
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -1,7 +1,6 @@
 stages:
   - test
   - docker-build
-  - oc-tag
   - deploy
 
 # Use the acc-py-devtools templates found at
@@ -11,39 +10,61 @@ include:
    file: acc_py_devtools/templates/gitlab-ci/python.yml
 
 variables:
-  project_name: caimira
-
+  PY_VERSION: "3.11"
 
 # ###################################################################################################
-# Test code
+# Test code - CAiMIRA (model) and CERN CAiMIRA (CERN's UI)
 
-# A full installation of CAiMIRA, tested with pytest.
-test_install:
-  extends: .acc_py_full_test
+.test-base:
+  image: registry.cern.ch/docker.io/library/python:${PY_VERSION}
+  stage: test
+  except:
+    - live/caimira-test  # do not run tests on live/caimira-test branch
 
+.test-run:
+  extends:
+    - .test-base
+  script:
+    - cd ./${PROJECT_ROOT}
+    - pip install -e .[test]
+    - python -m pytest
 
-# A development installation of CAiMIRA tested with pytest.
-test_dev:
-  extends: .acc_py_dev_test
+test-caimira-py311:
+  variables:
+    PROJECT_ROOT: "caimira"
+  extends:
+    - .test-run
 
+test-cern-caimira-py311:
+  before_script:
+    - cd ./caimira
+    - pip install -e .[test]
+    - cd ../
+  variables:
+    PROJECT_ROOT: "cern_caimira"
+  extends:
+    - .test-run
 
-# A development installation of CAiMIRA tested with pytest.
-test_dev-39:
+test-caimira-py39:
   variables:
     PY_VERSION: "3.9"
-  extends: .acc_py_dev_test
+    PROJECT_ROOT: "caimira"
+  extends:
+    - test-caimira-py311
 
+test-cern-caimira-py39:
+  variables:
+    PY_VERSION: "3.9"
+    PROJECT_ROOT: "cern_caimira"
+  extends:
+    - test-cern-caimira-py311
 
 # ###################################################################################################
 # Test OpenShift config
 
 .test_openshift_config:
   stage: test
-  rules:
-    - if: '$OC_TOKEN && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == $BRANCH'
-      allow_failure: true  # The branch must represent what is deployed.  FIXME: change to true because of a diff between ConfigMaps
-    - if: '$OC_TOKEN && $CI_MERGE_REQUEST_EVENT_TYPE != "detached"'
-      allow_failure: true   # Anything other than the branch may fail without blocking the pipeline.
+  allow_failure: true
   image: registry.cern.ch/docker.io/mambaorg/micromamba
   before_script:
     - micromamba create --yes -p $HOME/env python=3.9 ruamel.yaml wget -c conda-forge
@@ -63,7 +84,9 @@ test_dev-39:
     paths:
       - ./app-config/openshift/${CAIMIRA_INSTANCE}/actual
       - ./app-config/openshift/${CAIMIRA_INSTANCE}/expected
-
+  only:
+    - master
+    - live/caimira-test  # do not run tests on live/caimira-test branch
 
 check_openshift_config_test:
   extends: .test_openshift_config
@@ -73,35 +96,29 @@ check_openshift_config_test:
     OC_SERVER: https://api.paas.okd.cern.ch
     OC_TOKEN: "${OPENSHIFT_CAIMIRA_TEST_CONFIG_CHECKER_TOKEN}"
 
-
-check_openshift_config_prod:
-  extends: .test_openshift_config
-  variables:
-    CAIMIRA_INSTANCE: 'caimira-prod'
-    BRANCH: 'master'
-    OC_SERVER: https://api.paas.okd.cern.ch
-    OC_TOKEN: "${OPENSHIFT_CAIMIRA_PROD_CONFIG_CHECKER_TOKEN}"
+# TODO: for prod, it should ignore the different tag in the `image` field
+# check_openshift_config_prod:
+#   extends: .test_openshift_config
+#   variables:
+#     CAIMIRA_INSTANCE: 'caimira-prod'
+#     BRANCH: 'master'
+#     OC_SERVER: https://api.paas.okd.cern.ch
+#     OC_TOKEN: "${OPENSHIFT_CAIMIRA_PROD_CONFIG_CHECKER_TOKEN}"
 
 
 # ###################################################################################################
 # Build docker images
 
-.image_builder:
-  # Build and push images to the openshift instance, which automatically triggers an application re-deployment.
+# base
+.docker-build:
   stage: docker-build
-  rules:
-    - if: '$CI_COMMIT_BRANCH == "live/caimira-test"'
-      variables:
-        IMAGE_TAG: caimira-test-latest
-    - if: '$CI_COMMIT_BRANCH == "master"'
-      variables:
-        IMAGE_TAG: caimira-prod-latest
   image:
     # Based on guidance at https://gitlab.cern.ch/gitlabci-examples/build_docker_image.
     # The kaniko debug image is recommended because it has a shell, and a shell is required for an image to be used with GitLab CI/CD.
     name: gcr.io/kaniko-project/executor:debug
     entrypoint: [""]
   script:
+    - echo "Building image for ${CI_COMMIT_REF_NAME} branch with tag ${IMAGE_TAG}"
     # Prepare Kaniko configuration file
     - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
     - echo "Building ${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:latest Docker image..."
@@ -110,85 +127,95 @@ check_openshift_config_prod:
     # Print the full registry path of the pushed image
     - echo "Image pushed successfully to ${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${IMAGE_TAG}"
 
-
-auth-service-image_builder:
-  extends:
-    - .image_builder
+.docker-build-auth-service:
   variables:
     IMAGE_NAME: auth-service
     DOCKERFILE_DIRECTORY: app-config/auth-service
     DOCKER_CONTEXT_DIRECTORY: app-config/auth-service
+  extends: .docker-build
 
-
-calculator-app-image_builder:
-  extends:
-    - .image_builder
+.docker-build-calculator-app:
   variables:
     IMAGE_NAME: calculator-app
     DOCKERFILE_DIRECTORY: app-config/calculator-app
     DOCKER_CONTEXT_DIRECTORY: ""
+  extends: .docker-build
 
-
-oci_calculator:
-  extends: .image_builder
+# on push to live/caimira-test
+.docker-build-test:
   variables:
-    IMAGE_NAME: calculator
-    DOCKERFILE_DIRECTORY: app-config/caimira-public-docker-image
-    DOCKER_CONTEXT_DIRECTORY: ""
-
-
-# ###################################################################################################
-# Link build Docker images OpenShift <-> GitLab registry
-
-.link_docker_images_with_gitlab_registry:
-  stage: oc-tag
-  image: gitlab-registry.cern.ch/paas-tools/openshift-client:latest
-  rules:
-    - if: '$CI_COMMIT_BRANCH == "live/caimira-test"'
-      variables:
-        OC_PROJECT: "caimira-test"
-        OC_TOKEN: ${OPENSHIFT_CAIMIRA_TEST_DEPLOY_TOKEN}
-        IMAGE_TAG: caimira-test-latest
-    - if: '$CI_COMMIT_BRANCH == "master"'
-      variables:
-        OC_PROJECT: "caimira-prod"
-        OC_TOKEN: ${OPENSHIFT_CAIMIRA_PROD_DEPLOY_TOKEN}
-        IMAGE_TAG: caimira-prod-latest
-  script:
-     - oc tag --source=docker ${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${IMAGE_TAG} ${IMAGE_NAME}:latest --token ${OC_TOKEN} --server=https://api.paas.okd.cern.ch -n ${OC_PROJECT}
+    IMAGE_TAG: caimira-test-latest
 
-link_auth-service_with_gitlab_registry:
+docker-build-auth-service-test:
   extends:
-    - .link_docker_images_with_gitlab_registry
-  variables:
-    IMAGE_NAME: auth-service
+    - .docker-build-test
+    - .docker-build-auth-service
+  only:
+    - live/caimira-test
 
-link_calculator-app_with_gitlab_registry:
+docker-build-calculator-app-test:
   extends:
-    - .link_docker_images_with_gitlab_registry
-  variables:
-    IMAGE_NAME: calculator-app
+    - .docker-build-test
+    - .docker-build-calculator-app
+  only:
+    - live/caimira-test
 
-link_calculator_with_gitlab_registry:
+# on release
+.docker-build-release:
+  before_script:
+    # Extract version number without 'v' prefix as IMAGE_TAG
+    - IMAGE_TAG=$(echo "$CI_COMMIT_REF_NAME" | sed 's/^v//')
+    - echo "Version is $IMAGE_TAG"
+
+docker-build-auth-service-release:
   extends:
-    - .link_docker_images_with_gitlab_registry
-  variables:
-    IMAGE_NAME: calculator
+    - .docker-build-release
+    - .docker-build-auth-service
+  only:
+    - tag
 
+docker-build-calculator-app-release:
+  extends:
+    - .docker-build-release
+    - .docker-build-calculator-app
+  only:
+    - tag
 
 # ###################################################################################################
-# Trigger build of CAiMIRA router on OpenShift
-
-trigger_caimira-router_build_on_openshift:
+# Deploy to OpenShift
+.deploy:
   stage: deploy
-  rules:
-    - if: '$CI_COMMIT_BRANCH == "live/caimira-test"'
-      variables:
-        OC_PROJECT: "caimira-test"
-        BUILD_WEBHOOK_SECRET: ${OPENSHIFT_CAIMIRA_TEST_BUILD_WEBHOOK_SECRET}
-    - if: '$CI_COMMIT_BRANCH == "master"'
-      variables:
-        OC_PROJECT: "caimira-prod"
-        BUILD_WEBHOOK_SECRET: ${OPENSHIFT_CAIMIRA_PROD_BUILD_WEBHOOK_SECRET}
+  image: gitlab-registry.cern.ch/paas-tools/openshift-client
+  variables:
+    IMAGE_TAG: caimira-test-latest
+    OPENSHIFT_SERVER: https://api.paas.okd.cern.ch
+    OPENSHIFT_PROJECT: caimira-test
   script:
-     - curl -X POST -k https://api.paas.okd.cern.ch/apis/build.openshift.io/v1/namespaces/${OC_PROJECT}/buildconfigs/caimira-router/webhooks/${BUILD_WEBHOOK_SECRET}/generic
+    - echo "Deploying ${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${IMAGE_TAG} to OpenShift"
+    - oc login $OPENSHIFT_SERVER --token=$OPENSHIFT_CAIMIRA_TEST_DEPLOY_TOKEN
+    - oc project $OPENSHIFT_PROJECT
+    - oc set image dc/$OPENSHIFT_DEPLOYMENT $OPENSHIFT_CONTAINER_NAME=${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${IMAGE_TAG}
+    - oc rollout status dc/$OPENSHIFT_DEPLOYMENT
+  only:
+    - live/caimira-test
+
+deploy-auth-service-test:
+  extends: .deploy
+  variables:
+    IMAGE_NAME: auth-service
+    OPENSHIFT_DEPLOYMENT: auth-service
+    OPENSHIFT_CONTAINER_NAME: auth-service
+
+deploy-calculator-app-test:
+  extends: .deploy
+  variables:
+    IMAGE_NAME: calculator-app
+    OPENSHIFT_DEPLOYMENT: calculator-app
+    OPENSHIFT_CONTAINER_NAME: calculator-app
+
+deploy-calculator-open-app-test:
+  extends: .deploy
+  variables:
+    IMAGE_NAME: calculator-app
+    OPENSHIFT_DEPLOYMENT: calculator-open-app
+    OPENSHIFT_CONTAINER_NAME: calculator-open-app