Filters bugfix; add metrics and filter to logged sample
#3805
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tasks Modified | |
| on: | |
| push: | |
| branches: | |
| - 'main' | |
| pull_request: | |
| branches: | |
| - 'main' | |
| workflow_dispatch: | |
| # comment/edit out the above to stop/change the triggers | |
| jobs: | |
| changed_files: | |
| runs-on: ubuntu-latest # windows-latest || macos-latest | |
| timeout-minutes: 120 | |
| name: Scan for changed tasks | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 # OR "2" -> To retrieve the preceding commit. | |
| # Uses the tj-actions/changed-files action to check for changes. | |
| # Outputs provided here: https://github.com/tj-actions/changed-files#outputs | |
| # The `files_yaml` input optionally takes a yaml string to specify filters, | |
| # and prepends the filter name to the standard output names. | |
| - name: Check task folders | |
| id: changed-tasks | |
| uses: tj-actions/[email protected] | |
| with: | |
| # tasks checks the tasks folder and api checks the api folder for changes | |
| files_yaml: | | |
| tasks: | |
| - lm_eval/tasks/** | |
| api: | |
| - lm_eval/api/** | |
| write_output_files: true | |
| # The next step is optional; the files are written to the workspace by default (above). | |
| # so it's just for debugging | |
| - name: Run Tests | |
| if: steps.changed-tasks.outputs.tasks_any_modified == 'true' || steps.changed-tasks.outputs.api_any_modified == 'true' | |
| run: | | |
| echo .github/outputs/tasks_all_changed_and_modified_files.txt >> 'GITHUB_ENV' | |
| echo "One or more test file(s) has changed." | |
| echo "List of all the files that have changed: ${{ steps.changed-tasks.outputs.tasks_all_modified_files }}" | |
| - name: Set up Python 3.9 | |
| if: steps.changed-tasks.outputs.tasks_any_modified == 'true' || steps.changed-tasks.outputs.api_any_modified == 'true' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.9 | |
| cache: 'pip' | |
| cache-dependency-path: setup.py | |
| - name: Install dependencies | |
| if: steps.changed-tasks.outputs.tasks_any_modified == 'true' || steps.changed-tasks.outputs.api_any_modified == 'true' | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e '.[dev,ifeval]' --extra-index-url https://download.pytorch.org/whl/cpu | |
| # Install optional git dependencies | |
| # pip install bleurt@https://github.com/google-research/bleurt/archive/b610120347ef22b494b6d69b4316e303f5932516.zip#egg=bleurt | |
| # if [ -f requirements.txt ]; then pip install -r requirements.txt; fi | |
| - name: Test with pytest | |
| # if new tasks are added, run tests on them | |
| if: steps.changed-tasks.outputs.tasks_any_modified == 'true' | |
| run: python -m pytest tests/test_tasks.py -s -vv | |
| # if api is modified, run tests on it | |
| - name: Test more tasks with pytest | |
| env: | |
| API: true | |
| if: steps.changed-tasks.outputs.api_any_modified == 'true' | |
| run: python -m pytest tests/test_tasks.py -s -vv |