From 490c0dc03b0dd350086d1d707642187c3557d72d Mon Sep 17 00:00:00 2001 From: RohanSasne Date: Mon, 12 Jun 2023 00:00:43 +0530 Subject: [PATCH 01/11] Add yml file to run docker stack --- .github/workflows/docker-stack-test.yml | 130 ++++++++++++++++++++++++ docker-compose.yml | 1 + 2 files changed, 131 insertions(+) create mode 100644 .github/workflows/docker-stack-test.yml diff --git a/.github/workflows/docker-stack-test.yml b/.github/workflows/docker-stack-test.yml new file mode 100644 index 00000000000..1d3c13816eb --- /dev/null +++ b/.github/workflows/docker-stack-test.yml @@ -0,0 +1,130 @@ +name: CI + +on: + push: + branches: + - master + - develop + - Docker-Stack-Testing + + + tags: + - '*' + + pull_request: + + merge_group: + + issue_comment: + types: + - created + +env: + R_LIBS_USER: /usr/local/lib/R/site-library + LC_ALL: en_US.UTF-8 + NCPUS: 2 + PGHOST: postgres + CI: true + +jobs: + + + # ---------------------------------------------------------------------- + # SIPNET TESTS + # ---------------------------------------------------------------------- + sipnet: + if: github.event_name != 'issue_comment' || startsWith(github.event.comment.body, '/build') + runs-on: ubuntu-latest + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + + strategy: + fail-fast: false + + + + + + services: + postgres: + image: mdillon/postgis:9.5 + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + + container: + image: pecan/depends + + steps: + # checkout source code + - name: work around https://github.com/actions/checkout/issues/766 + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + - uses: actions/checkout@v3 + with: + set-safe-directory: false + + - name: Set up Docker Compose + run: | + sudo apt-get update + sudo apt-get -y install docker-compose + - name: Build and run containers + run: | + docker system prune --all --force + docker-compose up -d postgres + docker run --rm --network pecan_pecan pecan/db + + docker-compose run bety user guestuser guestuser "Guest User" guestuser@example.com 4 4 + + + docker-compose run bety user carya illinois "Carya Demo User" carya@example.com 1 1 + docker run --rm --network pecan_pecan --volume pecan_pecan:/data --env FQDN=docker pecan/data:develop + + cp docker/env.example .env + echo "COMPOSE_PROJECT_NAME=pecan" >> .env + echo "PECAN_VERSION=develop" >> .env + + docker-compose up -d + + + + - name: Wait for services to start + run: | + + docker-compose ps + sleep 20 # Adjust the duration as needed + docker-compose ps + sleep 20 + docker-compose ps + sleep 20 + + + + - name: Display running containers + run: | + docker-compose ps + docker network ls + docker network inspect bridge + + + # run SIPNET test + # run SIPNET test + - name: Run cURL command + run: | + curl -v -L -X POST -H "Host: pecan.localhost" \ + -F 'hostname=docker' \ + -F 'modelid=1000000014' \ + -F 'sitegroupid=1' \ + -F 'siteid=679' \ + -F 'sitename=Lost Creek LCR (US-Los)' \ + -F 'pft[]=luquillo.late_tropical' \ + -F 'start=2002/01/01' \ + -F 'end=2005/12/31' \ + -F 'input_met=lcr.clim' \ + -F 'email=' \ + -F 'notes=' \ + 'http://172.17.0.1/pecan/04-runpecan.php' + + + sleep 120 + curl -v -L -H "Host: pecan.localhost" \ + 'http://172.17.0.1/pecan/historylist.php' + + diff --git a/docker-compose.yml b/docker-compose.yml index 180c5653834..a093b940ba0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -300,6 +300,7 @@ services: api: image: pecan/api:${PECAN_VERSION:-latest} user: "${UID:-1001}:${GID:-1001}" + restart: unless-stopped networks: - pecan environment: From 211178153bee064580e2d6cf8655900206da3d1c Mon Sep 17 00:00:00 2001 From: RohanSasne Date: Mon, 12 Jun 2023 00:17:01 +0530 Subject: [PATCH 02/11] Add matrix defination to run parallel models --- .github/workflows/docker-stack-test.yml | 31 ++++++++++++++++++------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docker-stack-test.yml b/.github/workflows/docker-stack-test.yml index 1d3c13816eb..11b6414eead 100644 --- a/.github/workflows/docker-stack-test.yml +++ b/.github/workflows/docker-stack-test.yml @@ -1,4 +1,4 @@ -name: CI +name: Docker-Stack-Test on: push: @@ -30,9 +30,9 @@ jobs: # ---------------------------------------------------------------------- - # SIPNET TESTS + # DOCKER STACK TESTS # ---------------------------------------------------------------------- - sipnet: + Stack-Test: if: github.event_name != 'issue_comment' || startsWith(github.event.comment.body, '/build') runs-on: ubuntu-latest env: @@ -40,6 +40,19 @@ jobs: strategy: fail-fast: false + matrix: + name: + - sipnet_r136 + + include: + - name: sipnet_r136 + modelID: "1000000014" + siteID: "772" + siteName: "Niwot Ridge Forest/LTER NWT1 (US-NR1)" + pftName: "temperate.coniferous" + metData: "AmerifluxLBL.SIPNET" + + #Add more models below in the above format @@ -110,14 +123,14 @@ jobs: run: | curl -v -L -X POST -H "Host: pecan.localhost" \ -F 'hostname=docker' \ - -F 'modelid=1000000014' \ + -F 'modelid=${{ matrix.modelID }}' \ -F 'sitegroupid=1' \ - -F 'siteid=679' \ - -F 'sitename=Lost Creek LCR (US-Los)' \ - -F 'pft[]=luquillo.late_tropical' \ + -F 'siteid=${{ matrix.siteID }}' \ + -F 'sitename=${{ matrix.siteName }}' \ + -F 'pft[]=${{ matrix.pftName }}' \ -F 'start=2002/01/01' \ -F 'end=2005/12/31' \ - -F 'input_met=lcr.clim' \ + -F 'input_met=${{ matrix.metData }}' \ -F 'email=' \ -F 'notes=' \ 'http://172.17.0.1/pecan/04-runpecan.php' @@ -128,3 +141,5 @@ jobs: 'http://172.17.0.1/pecan/historylist.php' + + \ No newline at end of file From c1dce2f4292824e83ea83bd82c6dc0efcdac220a Mon Sep 17 00:00:00 2001 From: RohanSasne Date: Mon, 12 Jun 2023 00:38:50 +0530 Subject: [PATCH 03/11] Code cleanup --- .github/workflows/docker-stack-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-stack-test.yml b/.github/workflows/docker-stack-test.yml index 11b6414eead..bd2b7f3cb24 100644 --- a/.github/workflows/docker-stack-test.yml +++ b/.github/workflows/docker-stack-test.yml @@ -117,8 +117,8 @@ jobs: docker network inspect bridge - # run SIPNET test - # run SIPNET test + + # run tests against the full docker stack - name: Run cURL command run: | curl -v -L -X POST -H "Host: pecan.localhost" \ From 5a917b492ddfdf3ca917a6652a10e072d0f594c0 Mon Sep 17 00:00:00 2001 From: RohanSasne Date: Mon, 12 Jun 2023 14:41:55 +0530 Subject: [PATCH 04/11] Update Model Parameters --- .github/workflows/docker-stack-test.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docker-stack-test.yml b/.github/workflows/docker-stack-test.yml index bd2b7f3cb24..50a59c86efb 100644 --- a/.github/workflows/docker-stack-test.yml +++ b/.github/workflows/docker-stack-test.yml @@ -47,10 +47,10 @@ jobs: include: - name: sipnet_r136 modelID: "1000000014" - siteID: "772" + siteID: "678" siteName: "Niwot Ridge Forest/LTER NWT1 (US-NR1)" - pftName: "temperate.coniferous" - metData: "AmerifluxLBL.SIPNET" + pftName: "luquillo.late_tropical" + metData: "5000000003" #Add more models below in the above format @@ -136,10 +136,11 @@ jobs: 'http://172.17.0.1/pecan/04-runpecan.php' - sleep 120 - curl -v -L -H "Host: pecan.localhost" \ - 'http://172.17.0.1/pecan/historylist.php' - + curl -v -L -H "Host: pecan.localhost" \ + 'http://172.17.0.1/pecan/08-finished.php?workflowid=99000000001&hostname=docker&loglines=100' - \ No newline at end of file + sleep 200 + curl -v -L -H "Host: pecan.localhost" \ + 'http://172.17.0.1/pecan/08-finished.php?workflowid=99000000001&hostname=docker&loglines=100' + \ No newline at end of file From 97a955531b214ec75282ffde3890766b602afe05 Mon Sep 17 00:00:00 2001 From: RohanSasne Date: Mon, 12 Jun 2023 17:12:00 +0530 Subject: [PATCH 05/11] Add SIPNET output --- .github/workflows/docker-stack-test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-stack-test.yml b/.github/workflows/docker-stack-test.yml index 50a59c86efb..0c4598c1ca4 100644 --- a/.github/workflows/docker-stack-test.yml +++ b/.github/workflows/docker-stack-test.yml @@ -143,4 +143,6 @@ jobs: sleep 200 curl -v -L -H "Host: pecan.localhost" \ 'http://172.17.0.1/pecan/08-finished.php?workflowid=99000000001&hostname=docker&loglines=100' - \ No newline at end of file + + curl -v -L -H "Host: pecan.localhost" \ + 'http://172.17.0.1/pecan/dataset.php?workflowid=99000000001&type=file&name=out/99000000001/sipnet.out' \ No newline at end of file From 409e8625195df61085e3b1a4608265353e72c03f Mon Sep 17 00:00:00 2001 From: RohanSasne Date: Mon, 12 Jun 2023 19:21:15 +0530 Subject: [PATCH 06/11] Update model parameters --- .github/workflows/docker-stack-test.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker-stack-test.yml b/.github/workflows/docker-stack-test.yml index 0c4598c1ca4..3730aa0d662 100644 --- a/.github/workflows/docker-stack-test.yml +++ b/.github/workflows/docker-stack-test.yml @@ -47,10 +47,12 @@ jobs: include: - name: sipnet_r136 modelID: "1000000014" - siteID: "678" + siteID: "772" siteName: "Niwot Ridge Forest/LTER NWT1 (US-NR1)" - pftName: "luquillo.late_tropical" - metData: "5000000003" + pftName: "temperate.deciduous" + metData: "AmerifluxLBL" + startDate: "2004/01/01" + endDate: "2004/12/31" #Add more models below in the above format @@ -128,8 +130,8 @@ jobs: -F 'siteid=${{ matrix.siteID }}' \ -F 'sitename=${{ matrix.siteName }}' \ -F 'pft[]=${{ matrix.pftName }}' \ - -F 'start=2002/01/01' \ - -F 'end=2005/12/31' \ + -F 'start=${{ matrix.startDate }}' \ + -F 'end=${{ matrix.endDate }}' \ -F 'input_met=${{ matrix.metData }}' \ -F 'email=' \ -F 'notes=' \ @@ -142,7 +144,7 @@ jobs: sleep 200 curl -v -L -H "Host: pecan.localhost" \ - 'http://172.17.0.1/pecan/08-finished.php?workflowid=99000000001&hostname=docker&loglines=100' + 'http://172.17.0.1/pecan/08-finished.php?workflowid=99000000220&hostname=docker&loglines=100' curl -v -L -H "Host: pecan.localhost" \ 'http://172.17.0.1/pecan/dataset.php?workflowid=99000000001&type=file&name=out/99000000001/sipnet.out' \ No newline at end of file From 237ce47be87f8119c30c02115f5ba69d440b3419 Mon Sep 17 00:00:00 2001 From: RohanSasne Date: Mon, 12 Jun 2023 21:36:34 +0530 Subject: [PATCH 07/11] Update Matrix config --- .github/workflows/docker-stack-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-stack-test.yml b/.github/workflows/docker-stack-test.yml index 3730aa0d662..2a2efc86f36 100644 --- a/.github/workflows/docker-stack-test.yml +++ b/.github/workflows/docker-stack-test.yml @@ -144,7 +144,7 @@ jobs: sleep 200 curl -v -L -H "Host: pecan.localhost" \ - 'http://172.17.0.1/pecan/08-finished.php?workflowid=99000000220&hostname=docker&loglines=100' + 'http://172.17.0.1/pecan/08-finished.php?workflowid=99000000001&hostname=docker&loglines=100' curl -v -L -H "Host: pecan.localhost" \ 'http://172.17.0.1/pecan/dataset.php?workflowid=99000000001&type=file&name=out/99000000001/sipnet.out' \ No newline at end of file From c95dc05491448899f3dc1cec6f8e2e200648de04 Mon Sep 17 00:00:00 2001 From: Rohan Sasne Date: Thu, 15 Jun 2023 02:25:22 +0530 Subject: [PATCH 08/11] Update Model Parameters and Create Script file --- .github/workflows/docker-stack-test.yml | 69 +++++-------------------- tests/Docker-Stack-Test.sh | 18 +++++++ 2 files changed, 31 insertions(+), 56 deletions(-) create mode 100644 tests/Docker-Stack-Test.sh diff --git a/.github/workflows/docker-stack-test.yml b/.github/workflows/docker-stack-test.yml index 2a2efc86f36..df06f8fcd30 100644 --- a/.github/workflows/docker-stack-test.yml +++ b/.github/workflows/docker-stack-test.yml @@ -6,6 +6,7 @@ on: - master - develop - Docker-Stack-Testing + - new-branch-for-docker-testing tags: @@ -33,32 +34,14 @@ jobs: # DOCKER STACK TESTS # ---------------------------------------------------------------------- Stack-Test: - if: github.event_name != 'issue_comment' || startsWith(github.event.comment.body, '/build') + runs-on: ubuntu-latest env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} strategy: fail-fast: false - matrix: - name: - - sipnet_r136 - - include: - - name: sipnet_r136 - modelID: "1000000014" - siteID: "772" - siteName: "Niwot Ridge Forest/LTER NWT1 (US-NR1)" - pftName: "temperate.deciduous" - metData: "AmerifluxLBL" - startDate: "2004/01/01" - endDate: "2004/12/31" - - #Add more models below in the above format - - - - + services: postgres: @@ -83,23 +66,25 @@ jobs: - name: Build and run containers run: | docker system prune --all --force + cp docker/env.example .env + echo "COMPOSE_PROJECT_NAME=pecan" >> .env + echo "PECAN_VERSION=develop" >> .env + echo "UID=$(id -u)" >> .env + echo "GID=$(id -g)" >> .env docker-compose up -d postgres docker run --rm --network pecan_pecan pecan/db docker-compose run bety user guestuser guestuser "Guest User" guestuser@example.com 4 4 - docker-compose run bety user carya illinois "Carya Demo User" carya@example.com 1 1 docker run --rm --network pecan_pecan --volume pecan_pecan:/data --env FQDN=docker pecan/data:develop + docker run --rm --network pecan_pecan --volume pecan_pecan:/data pecan/data:develop chown -R "$(id -u).$(id -g)" /data - cp docker/env.example .env - echo "COMPOSE_PROJECT_NAME=pecan" >> .env - echo "PECAN_VERSION=develop" >> .env + docker run --user="$(id -u)" --rm --network pecan_pecan --volume pecan_pecan:/data --env FQDN=docker pecan/data:develop docker-compose up -d - - name: Wait for services to start run: | @@ -109,7 +94,6 @@ jobs: sleep 20 docker-compose ps sleep 20 - - name: Display running containers @@ -117,34 +101,7 @@ jobs: docker-compose ps docker network ls docker network inspect bridge - - - + # run tests against the full docker stack - - name: Run cURL command - run: | - curl -v -L -X POST -H "Host: pecan.localhost" \ - -F 'hostname=docker' \ - -F 'modelid=${{ matrix.modelID }}' \ - -F 'sitegroupid=1' \ - -F 'siteid=${{ matrix.siteID }}' \ - -F 'sitename=${{ matrix.siteName }}' \ - -F 'pft[]=${{ matrix.pftName }}' \ - -F 'start=${{ matrix.startDate }}' \ - -F 'end=${{ matrix.endDate }}' \ - -F 'input_met=${{ matrix.metData }}' \ - -F 'email=' \ - -F 'notes=' \ - 'http://172.17.0.1/pecan/04-runpecan.php' - - - - curl -v -L -H "Host: pecan.localhost" \ - 'http://172.17.0.1/pecan/08-finished.php?workflowid=99000000001&hostname=docker&loglines=100' - - sleep 200 - curl -v -L -H "Host: pecan.localhost" \ - 'http://172.17.0.1/pecan/08-finished.php?workflowid=99000000001&hostname=docker&loglines=100' - - curl -v -L -H "Host: pecan.localhost" \ - 'http://172.17.0.1/pecan/dataset.php?workflowid=99000000001&type=file&name=out/99000000001/sipnet.out' \ No newline at end of file + - name: Run Integration Test + run: git update-index --chmod=+x ./tests/Docker-Stack-Test.sh diff --git a/tests/Docker-Stack-Test.sh b/tests/Docker-Stack-Test.sh new file mode 100644 index 00000000000..5cea601c7cf --- /dev/null +++ b/tests/Docker-Stack-Test.sh @@ -0,0 +1,18 @@ +curl -v -L -X POST -H "Host: pecan.localhost" \ + -F 'hostname=docker' \ + -F 'modelid=1000000014' \ + -F 'sitegroupid=1' \ + -F 'siteid=756' \ + -F 'sitename=Southern Pine (Subcategory), Duke Forest Pine ' \ + -F 'pft[]=temperate.coniferous' \ + -F 'start=2004/01/01' \ + -F 'end=2004/12/31' \ + -F 'input_met=CRUNCEP' \ + -F 'email=' \ + -F 'notes=' \ +'http://172.17.0.1/pecan/04-runpecan.php' + +sleep 200 + +curl -v -L -H "Host: pecan.localhost" \ +'http://172.17.0.1/pecan/dataset.php?workflowid=99000000001&type=file&name=out/99000000001/sipnet.out' \ No newline at end of file From f27780a9889da912b7c545e0d6542ac42ee08105 Mon Sep 17 00:00:00 2001 From: Rohan Sasne Date: Thu, 15 Jun 2023 03:13:02 +0530 Subject: [PATCH 09/11] Update File Permission --- .github/workflows/docker-stack-test.yml | 5 ++++- tests/Docker-Stack-Test.sh | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-stack-test.yml b/.github/workflows/docker-stack-test.yml index df06f8fcd30..2c15eb7fe27 100644 --- a/.github/workflows/docker-stack-test.yml +++ b/.github/workflows/docker-stack-test.yml @@ -104,4 +104,7 @@ jobs: # run tests against the full docker stack - name: Run Integration Test - run: git update-index --chmod=+x ./tests/Docker-Stack-Test.sh + run: | + chmod +x /tests/Docker-Stack-Test.sh + ./tests/Docker-Stack-Test.sh + diff --git a/tests/Docker-Stack-Test.sh b/tests/Docker-Stack-Test.sh index 5cea601c7cf..c38103dec11 100644 --- a/tests/Docker-Stack-Test.sh +++ b/tests/Docker-Stack-Test.sh @@ -3,7 +3,7 @@ curl -v -L -X POST -H "Host: pecan.localhost" \ -F 'modelid=1000000014' \ -F 'sitegroupid=1' \ -F 'siteid=756' \ - -F 'sitename=Southern Pine (Subcategory), Duke Forest Pine ' \ + -F 'sitename=Duke Forest - loblolly pine (US-Dk3)' \ -F 'pft[]=temperate.coniferous' \ -F 'start=2004/01/01' \ -F 'end=2004/12/31' \ @@ -15,4 +15,4 @@ curl -v -L -X POST -H "Host: pecan.localhost" \ sleep 200 curl -v -L -H "Host: pecan.localhost" \ -'http://172.17.0.1/pecan/dataset.php?workflowid=99000000001&type=file&name=out/99000000001/sipnet.out' \ No newline at end of file +'http://172.17.0.1/pecan/dataset.php?workflowid=99000000001&type=file&name=out/99000000001/sipnet.out' From 35ad66c513d4f7c55a775b9bdf9cba7e0c37766b Mon Sep 17 00:00:00 2001 From: Rohan Sasne Date: Thu, 15 Jun 2023 04:15:05 +0530 Subject: [PATCH 10/11] Fix File Permission --- .github/workflows/docker-stack-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-stack-test.yml b/.github/workflows/docker-stack-test.yml index 2c15eb7fe27..5c58e1452ca 100644 --- a/.github/workflows/docker-stack-test.yml +++ b/.github/workflows/docker-stack-test.yml @@ -105,6 +105,6 @@ jobs: # run tests against the full docker stack - name: Run Integration Test run: | - chmod +x /tests/Docker-Stack-Test.sh + chmod +x ./tests/Docker-Stack-Test.sh ./tests/Docker-Stack-Test.sh From a4e1421ef02db76865d273eaf59e20f28bbbe307 Mon Sep 17 00:00:00 2001 From: Rohan Sasne Date: Thu, 15 Jun 2023 04:42:34 +0530 Subject: [PATCH 11/11] Set Test to Every Monday and Add Template for cURL Request --- .github/workflows/docker-stack-test.yml | 21 ++++-------------- tests/Docker-Stack-Test.sh | 29 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/.github/workflows/docker-stack-test.yml b/.github/workflows/docker-stack-test.yml index 5c58e1452ca..b76dc605d3c 100644 --- a/.github/workflows/docker-stack-test.yml +++ b/.github/workflows/docker-stack-test.yml @@ -1,24 +1,11 @@ name: Docker-Stack-Test on: - push: - branches: - - master - - develop - - Docker-Stack-Testing - - new-branch-for-docker-testing - - - tags: - - '*' - - pull_request: - - merge_group: + # every Monday at 4:30 AM + + schedule: + - cron: '30 4 * * 1' - issue_comment: - types: - - created env: R_LIBS_USER: /usr/local/lib/R/site-library diff --git a/tests/Docker-Stack-Test.sh b/tests/Docker-Stack-Test.sh index c38103dec11..61a31fdb7e0 100644 --- a/tests/Docker-Stack-Test.sh +++ b/tests/Docker-Stack-Test.sh @@ -16,3 +16,32 @@ sleep 200 curl -v -L -H "Host: pecan.localhost" \ 'http://172.17.0.1/pecan/dataset.php?workflowid=99000000001&type=file&name=out/99000000001/sipnet.out' + + +# ----------------------------------------------- +# cURL Request Template +#----------------------------------------------- + +# Be sure to use the -H header and also use the bridge IP address of the Docker host (172.17.0.1) and not localhost + + +# curl -v -L -X POST -H "Host: pecan.localhost" \ +# -F 'hostname=' \ +# -F 'modelid=' \ +# -F 'sitegroupid=' \ +# -F 'siteid=' \ +# -F 'sitename=' \ +# -F 'pft[]=' \ +# -F 'start=' \ +# -F 'end=' \ +# -F 'input_met=' \ +# -F 'email=' \ +# -F 'notes=' \ +# 'http://172.17.0.1/pecan/04-runpecan.php' + +# Adjust the sleep time according to the model you are running + +# Append the workflowid and ID after `out/` by 1 for every consecutive run + +# curl -v -L -H "Host: pecan.localhost" \ +# 'http://172.17.0.1/pecan/dataset.php?workflowid=(ID)&type=file&name=out/(ID)/sipnet.out' \ No newline at end of file