Branch 2.1

.github/CODEOWNERS

@@ -0,0 +1 @@
+/.github/workflows/* @glorysdj @liu-shaojun @intel-analytics/CICD

.github/actions/create-job-status-badge/action.yml

@@ -0,0 +1,72 @@
+name: 'Create Job Status Badge'
+description: 'In a workflow with multiple jobs, create a badge that can display the completion time and status of the job'
+inputs:
+  secret:
+    description: 'use to access gist'
+    required: true
+  gist-id:
+    description: 'gist-id'
+    required: true
+  file-name:
+    description: 'gist file name(.json)'
+    required: true
+  type:
+    description: 'workflow or job'
+    required: true
+  job-name:
+    description: 'job name'
+  is-self-hosted-runner:
+    description: 'If use self-hosted runner it will be true'
+    required: true
+    default: false
+  runner-hosted-on:
+    description: 'Select the region of the runner host to set the proxy'
+    required: true
+    default: ''
+runs:
+  using: "composite"
+  steps:
+    - name: print time
+      shell: bash
+      run: |
+        export NOW=$( date '+%F_%H:%M:%S' )
+        echo "TIME=${NOW}" >> $GITHUB_ENV
+
+    - name: get job status
+      shell: bash
+      run: |
+        echo "JOB-STATUS=${{ job.status }}" >>$GITHUB_ENV
+    - name: set badge color
+      shell: bash
+      run: if [ ${{ env.JOB-STATUS }} == "success" ] ; then echo "COLOR=green" >> $GITHUB_ENV ; else echo "COLOR=red" >> $GITHUB_ENV; fi
+
+    - name: Create job badge
+      if: ${{ inputs.type == 'job' && inputs.secret != '' }}
+      uses: analytics-zoo/dynamic-badges-action@master
+      with:
+        auth: ${{ inputs.secret }}
+        gistID: ${{ inputs.gist-id }}
+        isSelfHostedRunner: ${{ inputs.is-self-hosted-runner }}
+        filename: ${{ inputs.file-name }}
+        label: ${{ inputs.job-name }}
+        message: ${{ env.JOB-STATUS }}
+        color: ${{ env.COLOR }}
+        runnerHostedOn: ${{ inputs.runner-hosted-on }}
+
+    - name: Create time badge
+      if: ${{ inputs.type == 'workflow' && inputs.secret != '' }}
+      uses: analytics-zoo/dynamic-badges-action@master
+      with:
+        auth: ${{ inputs.secret }}
+        gistID: ${{ inputs.gist-id }}
+        isSelfHostedRunner: ${{ inputs.is-self-hosted-runner }}
+        filename: ${{ inputs.file-name }}
+        label: Time is
+        message: ${{ env.TIME }}
+        runnerHostedOn: ${{ inputs.runner-hosted-on }}
+
+    - name: print error message
+      if: ${{ inputs.secret == '' }}
+      shell: bash
+      run: |
+        echo "Can not get secret! Please check out your workflow!(mainly caused by pr trigger)"

.github/actions/dllib-scala-ut-action/action.yml

@@ -0,0 +1,22 @@
+name: 'Run Dllib Scala UT'
+description: 'Run Dllib Scala UT'
+runs:
+  using: "composite"
+  steps:
+    - name: Run tests
+      shell: bash
+      run: |
+        export SPARK_LOCAL_HOSTNAME=localhost
+        export KERAS_BACKEND=tensorflow
+        cd scala
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.optim" "-Dtest=com.intel.analytics.bigdl.dllib.optim.*Test" test -P spark_3.x -Dspark.version=3.1.2 
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.models" "-Dtest=com.intel.analytics.bigdl.dllib.models.*Test" test -P spark_3.x -Dspark.version=3.1.2 
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.estimator" "-Dtest=com.intel.analytics.bigdl.dllib.estimator.*Test" test -P spark_3.x -Dspark.version=3.1.2  
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.nnframes" "-Dtest=com.intel.analytics.bigdl.dllib.nnframes.*Test" test -P spark_3.x -Dspark.version=3.1.2 
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.feature" "-Dtest=com.intel.analytics.bigdl.dllib.feature.*Test" test -P spark_3.x -Dspark.version=3.1.2
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.utils.intermediate" "-Dtest=com.intel.analytics.bigdl.dllib.utils.intermediate.*Test" test -P spark_3.x -Dspark.version=3.1.2
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.utils.tf" "-Dtest=com.intel.analytics.bigdl.dllib.utils.tf.*Test" test -P spark_3.x -Dspark.version=3.1.2
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.utils.python.api" "-Dtest=com.intel.analytics.bigdl.dllib.utils.python.api.*Test" test -P spark_3.x -Dspark.version=3.1.2
+        #need python requirements
+        #mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.keras" "-Dtest=com.intel.analytics.bigdl.dllib.keras.*Test" test -P spark_3.x -Dspark.version=3.1.2 
+        #mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.nn.mkldnn" "-Dtest=com.intel.analytics.bigdl.dllib.nn.mkldnn.*Test" test -P spark_3.x -Dspark.version=3.1.2 

.github/actions/friesian-scala-ut-action/action.yml

@@ -0,0 +1,42 @@
+name: 'Run Friesian Scala UT'
+description: 'Run Friesian Scala UT'
+runs:
+  using: "composite"
+  steps:
+    - name: Set up redis
+      shell: bash
+      run: |
+          sudo mkdir -p /usr/share/keyrings
+          curl -fsSL https://packages.redis.io/gpg | sudo -E gpg --yes --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg
+          echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list
+          sudo -E apt-get update
+          sudo -E apt-get install -y redis redis-tools
+          redis-server --daemonize yes
+          redis-cli ping
+
+    - name: Set up mkl
+      shell: bash
+      run: |
+        cd /tmp
+        curl -O http://10.239.45.10:8081/repository/raw/analytics-zoo-data/faiss-lib.zip
+        sudo unzip -o faiss-lib.zip -d /lib
+        sudo ln -fs /lib/libmkl_core.so /lib/libmkl_core.so.2
+        sudo ln -fs /lib/libmkl_gnu_thread.so /lib/libmkl_gnu_thread.so.2
+        sudo ln -fs /lib/libmkl_intel_lp64.so /lib/libmkl_intel_lp64.so.2
+
+    - name: Prepare data
+      shell: bash
+      run: |
+        mkdir -p /tmp/friesian-unittest
+        cd /tmp/friesian-unittest
+        curl -O http://10.239.45.10:8081/repository/raw/analytics-zoo-data/friesian-serving-upload.tar.gz 
+        tar -zxf friesian-serving-upload.tar.gz
+
+    - name: Run tests
+      shell: bash
+      run: |
+        export SPARK_LOCAL_HOSTNAME=localhost
+        export KERAS_BACKEND=tensorflow
+        cd scala
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.friesian" "-Dtest=com.intel.analytics.bigdl.friesian.*Test" test -P spark_2.x -Dspark.version=2.4.6 "-Dspark.master=local[*]"
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.friesian" "-Dtest=com.intel.analytics.bigdl.friesian.*Test" test -P spark_3.x -Dspark.version=3.1.2 "-Dspark.master=local[*]"

.github/actions/jdk-setup-action/action.yml

@@ -0,0 +1,11 @@
+name: 'Setup JDK8'
+description: 'setup jdk8'
+runs:
+  using: "composite"
+  steps:
+    - name: Set up JDK 8
+      uses: actions/setup-java@v3
+      with:
+        java-version: '8'
+        distribution: 'temurin'
+        settings-path: ${{ github.workspace }} # location for the settings.xml file

.github/actions/maven-setup-action/action.yml

@@ -0,0 +1,41 @@
+name: 'Setup Maven'
+description: 'setup maven'
+# inputs:
+#   arda_nexus:
+#     description: 'arda nexus secret'
+#     required: true
+runs:
+  using: "composite"
+  steps:
+    #pls use cache only on github owned runners 
+    # - uses: actions/cache@v2
+    #   with:
+    #     path: ~/.m2/repository
+    #     key: maven-${{ hashFiles('**/pom.xml') }}
+    #     restore-keys: maven-
+
+    - name: Set up Maven
+      uses: stCarolas/[email protected]
+      with:
+        maven-version: 3.8.2
+
+    - name: Set up Maven Settings
+      uses: s4u/[email protected]
+      with:
+        sonatypeSnapshots: true
+        apacheSnapshots: true
+        servers: |
+          [
+            {
+              "id": "central",
+              "configuration": {
+                "httpConfiguration": {
+                  "all": {
+                    "connectionTimeout": "3600000",
+                    "readTimeout": "3600000"
+                  }    
+                }
+              }
+            }
+          ]
+        mirrors: '[{"id": "ardaNexus", "name": "ardaNexus", "mirrorOf": "*", "url": "${NEXUS_URL}" }]'

.github/actions/orca-ray-ctx-example-action/action.yml

@@ -0,0 +1,44 @@
+name: 'Run Orca Ray Ctx Example'
+description: 'Run Orca Ray Ctx Example'
+runs:
+  using: "composite"
+  steps:
+    - name: Setup Env
+      shell: bash
+      run: |
+        apt-get update
+        apt-get install wget
+
+    - name: Set up Python 
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.7.11'
+
+    - name: Install dependencies
+      shell: bash
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install --upgrade setuptools==58.0.4
+
+        pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3
+        wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requirements/requirements-orca-python-ray.txt -O ${{ github.workspace }}/requirements-orca-python-ray.txt
+        pip uninstall -r ${{ github.workspace }}/requirements-orca-python-ray.txt -y
+        pip install -i https://pypi.tuna.tsinghua.edu.cn/simple -r ${{ github.workspace }}/requirements-orca-python-ray.txt 
+        pip install pytest
+
+        pip install cmake==3.16.3
+        pip install tensorflow==2.3.0
+        HOROVOD_WITH_TENSORFLOW=1; HOROVOD_WITH_GLOO=1; pip install --no-cache-dir horovod==0.19.2
+
+        pip install ray==1.11.0
+        pip install --pre --upgrade bigdl-orca
+
+    - name: Run Test
+      shell: bash
+      run: |
+        export SPARK_LOCAL_HOSTNAME=localhost
+        chmod a+x python/orca/dev/example/run-example-test-ray-runtime.sh
+        python/orca/dev/example/run-example-test-ray-runtime.sh
+      env:
+        BIGDL_ROOT: ${{ github.workspace }}
+        ANALYTICS_ZOO_ROOT: ${{ github.workspace }}

.github/actions/orca/orca-jep-exampletest-action/nightly-test/action.yml

@@ -0,0 +1,24 @@
+name: 'Run Orca Jep ExampleTest'
+description: 'Run Orca Jep ExampleTest'
+runs:
+  using: "composite"
+  steps:
+    - name: Run Test
+      shell: bash
+      run: |
+        source activate py37
+        export SPARK_LOCAL_HOSTNAME=localhost
+        chmod a+x python/orca/dev/example/run-example-tests-jep.sh
+        chmod a+x apps/run-app-tests-pytorch-pip.sh
+        if [ -f /root/.cache/torch/hub/checkpoints/celebaHQ16_december_s7_i96000-9c72988c.pth ]
+        then
+          rm /root/.cache/torch/hub/checkpoints/celebaHQ16_december_s7_i96000-9c72988c.pth
+        fi
+        wget -P /root/.cache/torch/hub/checkpoints/ $HTTP_URI/analytics-zoo-models/face-generation/celebaHQ16_december_s7_i96000-9c72988c.pth
+
+        bash python/orca/dev/example/run-example-tests-jep.sh
+        bash apps/run-app-tests-pytorch-pip.sh
+        source deactivate
+      env:
+        BIGDL_ROOT: ${{ github.workspace }}
+        ANALYTICS_ZOO_ROOT: ${{ github.workspace }}

.github/actions/orca/orca-python-exampletest-action/nightly-test/action.yml

@@ -0,0 +1,18 @@
+name: 'Run Orca Python ExampleTest'
+description: 'Run Orca Python ExampleTest'
+runs:
+  using: "composite"
+  steps:
+    - name: Run Test
+      shell: bash
+      run: |
+        source activate py37
+        export MAVEN_OPTS="-XX:MaxPermSize=3G -Xmx100G -XX:ReservedCodeCacheSize=512m"
+        export _JAVA_OPTIONS="-Xmx100G -XX:MaxPermSize=3G"
+        export SPARK_LOCAL_HOSTNAME=localhost
+        chmod a+x python/orca/dev/example/run-example-tests.sh
+        python/orca/dev/example/run-example-tests.sh
+        source deactivate
+      env:
+        BIGDL_ROOT: ${{ github.workspace }}
+        ANALYTICS_ZOO_ROOT: ${{ github.workspace }}

.github/actions/orca/orca-python-exampletest-ray-action/nightly-test/action.yml

@@ -0,0 +1,16 @@
+name: 'Run Orca Python ExampleTest Ray'
+description: 'Run Orca Python ExampleTest Ray'
+runs:
+  using: "composite"
+  steps:
+    - name: Run Test
+      shell: bash
+      run: |
+        source activate py37
+        export SPARK_LOCAL_HOSTNAME=localhost
+        chmod a+x python/orca/dev/example/run-example-test-ray.sh
+        python/orca/dev/example/run-example-test-ray.sh
+        source deactivate
+      env:
+        BIGDL_ROOT: ${{ github.workspace }}
+        ANALYTICS_ZOO_ROOT: ${{ github.workspace }}

.github/actions/orca/orca-python-horovod-pytorch-py37-spark3-action/nightly-test/action.yml

@@ -0,0 +1,17 @@
+name: 'Run Orca Python Horovod Tf2 Py37 Spark3'
+description: 'Run Orca Python Horovod Tf2 Py37 Spark3'
+runs:
+  using: "composite"
+  steps:
+    - name: Run Test
+      shell: bash
+      run: |
+        source activate py37
+        export SPARK_LOCAL_HOSTNAME=localhost
+        chmod a+x python/orca/dev/test/run-pytests-horovod-tf.sh
+        python/orca/dev/test/run-pytests-horovod-tf.sh
+        ray stop
+        source deactivate
+      env:
+        BIGDL_ROOT: ${{ github.workspace }}
+        ANALYTICS_ZOO_ROOT: ${{ github.workspace }}

.github/actions/orca/orca-python-horovod-tf2-py37-spark3-action/nightly-test/action.yml

@@ -0,0 +1,17 @@
+name: 'Run Orca Python Horovod Tf2 Py37 Spark3'
+description: 'Run Orca Python Horovod Tf2 Py37 Spark3'
+runs:
+  using: "composite"
+  steps:
+    - name: Run Test
+      shell: bash
+      run: |
+        source activate py37
+        export SPARK_LOCAL_HOSTNAME=localhost
+        chmod a+x python/orca/dev/test/run-pytests-horovod-tf.sh
+        python/orca/dev/test/run-pytests-horovod-tf.sh
+        ray stop
+        source deactivate
+      env:
+        BIGDL_ROOT: ${{ github.workspace }}
+        ANALYTICS_ZOO_ROOT: ${{ github.workspace }}

.github/actions/orca/orca-python-py37-spark3-action/nightly-test/action.yml

@@ -0,0 +1,25 @@
+name: 'Run Orca Python Py37 Spark3'
+description: 'Run Orca Python Py37 Spark3'
+runs:
+  using: "composite"
+  steps:
+    - name: Run Test
+      shell: bash
+      run: |
+        source activate py37
+        export SPARK_LOCAL_HOSTNAME=localhost
+        sed -i "s/'bigdl-core=='+VERSION/'bigdl-core==2.1.0b20220811'/g" python/dllib/src/setup.py
+        bash python/dev/release_default_linux_spark312.sh default false false false -Ddata-store-url=$HTTP_URI -U
+
+        # install dllib
+        pip install python/dllib/src/dist/bigdl_dllib*-py3-none-manylinux1_x86_64.whl
+
+        # install orca
+        pip install python/orca/src/dist/bigdl_orca*-py3-none-manylinux1_x86_64.whl
+
+        chmod a+x python/orca/dev/test/run-pytests-spark.sh
+        python/orca/dev/test/run-pytests-spark.sh
+        source deactivate
+      env:
+        BIGDL_ROOT: ${{ github.workspace }}
+        ANALYTICS_ZOO_ROOT: ${{ github.workspace }}

.github/actions/orca/orca-python-ray-py37-spark3-action/nightly-test/action.yml

@@ -0,0 +1,16 @@
+name: 'Run Orca Python Ray Py37 Spark3'
+description: 'Run Orca Python Ray Py37 Spark3'
+runs:
+  using: "composite"
+  steps:
+    - name: Run Test
+      shell: bash
+      run: |
+        source activate py37
+        export SPARK_LOCAL_HOSTNAME=localhost
+        chmod a+x python/orca/dev/test/run-pytests-ray
+        python/orca/dev/test/run-pytests-ray
+        source deactivate
+      env:
+        BIGDL_ROOT: ${{ github.workspace }}
+        ANALYTICS_ZOO_ROOT: ${{ github.workspace }}