ci-testing.yml

name: CI testing

# see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
on: # Trigger the workflow on push or pull request, but only for the master branch
  push:
    branches: ["master", "release/*"]
  pull_request: {}

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}
  cancel-in-progress: ${{ ! (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release/')) }}

defaults:
  run:
    shell: bash

jobs:

  pytester:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      # max-parallel: 6
      matrix:
        # PyTorch 1.5 is failing on Win and bolts requires torchvision>=0.5
        os: [ubuntu-20.04, macOS-12, windows-2022]
        python-version: [3.8, 3.9]
        topic: ['core']
        extra: [[]]
        include:
          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'core', extra: [] }
          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'image', extra: ['image_extra'] }
          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'image', extra: ['image_baal'] }
          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'image', extra: ['image_segm'] }
          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'image', extra: ['image_vissl'] }
          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'video', extra: [] }
          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'tabular', extra: [] }
          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'text', extra: [] }
          - { os: 'ubuntu-20.04', python-version: 3.8, topic: 'pointcloud', extra: [] }
          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'serve', extra: [] }
          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'graph', extra: [] }
          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'audio', extra: [] }
          - { os: 'ubuntu-20.04', python-version: 3.8, topic: 'core', extra: [], requires: 'oldest' }
          - { os: 'ubuntu-20.04', python-version: 3.8, topic: 'image', extra: [], requires: 'oldest' }
          - { os: 'ubuntu-20.04', python-version: 3.8, topic: 'vision', extra: [], requires: 'oldest' }
          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'tabular', extra: [], requires: 'oldest' }
          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'text', extra: [], requires: 'oldest' }
          #- { os: 'ubuntu-20.04', python-version: 3.8, topic: 'serve', extra: [], requires: 'oldest' }  # todo

    # Timeout: https://stackoverflow.com/a/59076067/4521646
    timeout-minutes: 70
    env:
      FREEZE_REQUIREMENTS: 1
      TORCH_URL: https://download.pytorch.org/whl/cpu/torch_stable.html
      TRANSFORMERS_CACHE: _hf_cache
      DATASETS_VERBOSITY: warning

    steps:
    - uses: actions/checkout@v3
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

    - name: Set Swap Space
      if: runner.os == 'Linux'
      uses: pierotofy/set-swap-space@master
      with:
        swap-size-gb: 10

    # Github Actions: Run step on specific OS: https://stackoverflow.com/a/57948488/4521646
    - name: Setup macOS
      if: runner.os == 'macOS'
      run: brew install libomp openblas lapack
    - name: Setup Ubuntu
      if: runner.os == 'Linux'
      run: sudo apt-get install -y libsndfile1 graphviz

    - name: Set min. dependencies
      if: matrix.requires == 'oldest'
      run: |
        import glob, os
        files = glob.glob(os.path.join("requirements", "*.txt"))
        for fname in files:
          lines = [line.replace('>=', '==') for line in open(fname).readlines()]
          open(fname, 'w').writelines(lines)
      shell: python

    - name: Adjust extras
      run: |
        import os
        extras = ['${{ matrix.topic }}'] + ${{ toJSON(matrix.extra) }}
        with open(os.getenv('GITHUB_ENV'), "a") as gh_env:
            gh_env.write(f"EXTRAS={','.join(extras)}")
      shell: python

    - name: Get pip cache dir
      id: pip-cache
      run: echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
    - name: Restore pip cache
      uses: actions/cache/restore@v3
      id: restore-cache
      with:
        path: ${{ steps.pip-cache.outputs.dir }}
        key: pip-dependencies

    - name: Install package
      run: |
        # todo: some dependency has not correct format of their extras
        python -m pip install "pip==22.3.1"
        # todo: this is a hack to be able to install packages that are checking torch version while install
        pip install numpy Cython "torch>=1.11.0" -f $TORCH_URL
        pip install .[$EXTRAS,test] --upgrade \
          --prefer-binary \
          -f $TORCH_URL \
          -f https://data.pyg.org/whl/torch-1.13.1+cpu.html  # this extra URL is for graph extras
        pip list

    - name: Restore HF cache
      uses: actions/cache/restore@v3
      with:
        path: ${{ env.TRANSFORMERS_CACHE }}
        key: cache-transformers

    - name: DocTests
      working-directory: src/
      run: |
        mv flash flashy
        pytest . --doctest-modules --doctest-plus
        mv flashy flash

    - name: Install dependencies
      run: |
        pip install .[$EXTRAS,test] \
          -r requirements/testing_${{ matrix.topic }}.txt  \
          --upgrade --prefer-binary -f $TORCH_URL
        pip cache info
        pip list

    - name: Save pip cache
      if: github.ref == 'refs/heads/master'
      uses: actions/cache/save@v3
      with:
        path: ${{ steps.pip-cache.outputs.dir }}
        key: pip-dependencies

    - name: Testing
      run: |
        coverage run --source flash -m pytest \
          tests/core \
          tests/deprecated_api \
          tests/examples \
          tests/template \
          tests/${{ matrix.topic }} \
          -v --timeout=300 --durations=50 # --reruns 3 --reruns-delay 2

    - name: Save HF cache
      if: github.ref == 'refs/heads/master'
      uses: actions/cache/save@v3
      with:
        path: ${{ env.TRANSFORMERS_CACHE }}
        key: cache-transformers

    - name: Statistics
      run: |
        coverage report
        coverage xml

    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v3
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        file: ./coverage.xml
        flags: unittests,${{ matrix.topic }},${{ matrix.extra }}
        env_vars: OS,PYTHON
        name: codecov-umbrella
        fail_ci_if_error: false


  testing-guardian:
    runs-on: ubuntu-latest
    needs: pytester
    if: always()
    steps:
    - run: echo "${{ needs.pytester.result }}"
    - name: failing...
      if: needs.pytester.result == 'failure'
      run: exit 1
    - name: cancelled or skipped...
      if: contains(fromJSON('["cancelled", "skipped"]'), needs.pytester.result)
      timeout-minutes: 1
      run: sleep 90