BLD, CI: Use cibuildwheel to build Emscripten/Pyodide wheels, push …
#8
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Workflow to build wheels for upload to PyPI. | |
| # Inspired by numpy's cibuildwheel config https://github.com/numpy/numpy/blob/main/.github/workflows/wheels.yml | |
| # | |
| # In an attempt to save CI resources, wheel builds do | |
| # not run on each push but only weekly and for releases. | |
| # Wheel builds can be triggered from the Actions page | |
| # (if you have the permissions) on a commit to main. | |
| # | |
| # Alternatively, you can add labels to the pull request in order to trigger wheel | |
| # builds. | |
| # The label(s) that trigger builds are: | |
| # - Build | |
| name: Wheel builder | |
| on: | |
| schedule: | |
| # 3:27 UTC every day | |
| - cron: "27 3 * * *" | |
| push: | |
| pull_request: | |
| types: [labeled, opened, synchronize, reopened] | |
| paths-ignore: | |
| - "doc/**" | |
| - "web/**" | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| jobs: | |
| build_sdist: | |
| name: Build sdist | |
| if: >- | |
| (github.event_name == 'schedule') || | |
| github.event_name == 'workflow_dispatch' || | |
| (github.event_name == 'pull_request' && | |
| contains(github.event.pull_request.labels.*.name, 'Build')) || | |
| (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && ( ! endsWith(github.ref, 'dev0'))) | |
| runs-on: ubuntu-22.04 | |
| env: | |
| IS_PUSH: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }} | |
| IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} | |
| outputs: | |
| sdist_file: ${{ steps.save-path.outputs.sdist_name }} | |
| steps: | |
| - name: Checkout pandas | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Build sdist | |
| run: | | |
| python -m pip install build | |
| python -m build --sdist | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: sdist | |
| path: ./dist/* | |
| - name: Sanity check sdist files | |
| run: | | |
| ls ./dist | |
| - name: Output sdist name | |
| id: save-path | |
| shell: bash -el {0} | |
| run: echo "sdist_name=$(ls ./dist)" >> "$GITHUB_OUTPUT" | |
| build_wheels: | |
| needs: build_sdist | |
| name: Build wheel for ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }} | |
| if: >- | |
| (github.event_name == 'schedule') || | |
| github.event_name == 'workflow_dispatch' || | |
| (github.event_name == 'pull_request' && | |
| contains(github.event.pull_request.labels.*.name, 'Build')) || | |
| (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && ( ! endsWith(github.ref, 'dev0'))) | |
| runs-on: ${{ matrix.buildplat[0] }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # GitHub Actions doesn't support pairing matrix values together, let's improvise | |
| # https://github.com/github/feedback/discussions/7835#discussioncomment-1769026 | |
| buildplat: | |
| - [ubuntu-22.04, manylinux_x86_64] | |
| - [ubuntu-22.04, musllinux_x86_64] | |
| - [macos-12, macosx_x86_64] | |
| # Note: M1 images on Github Actions start from macOS 14 | |
| - [macos-14, macosx_arm64] | |
| - [windows-2022, win_amd64] | |
| # TODO: support PyPy? | |
| python: [["cp310", "3.10"], ["cp311", "3.11"], ["cp312", "3.12"]] | |
| # Build Pyodide wheels and upload them to Anaconda.org | |
| # NOTE: this job is similar to the one in unit-tests.yml except for the fact | |
| # that it uses cibuildwheel instead of a standard Pyodide xbuildenv setup. | |
| include: | |
| - buildplat: [ubuntu-22.04, pyodide_wasm32] | |
| python: ["cp312", "3.12"] | |
| env: | |
| IS_PUSH: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }} | |
| IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} | |
| steps: | |
| - name: Checkout pandas | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| # TODO: Build wheels from sdist again | |
| # There's some sort of weird race condition? | |
| # within Github that makes the sdist be missing files | |
| # We need to build wheels from the sdist since the sdist | |
| # removes unnecessary files from the release | |
| - name: Download sdist (not macOS) | |
| #if: ${{ matrix.buildplat[1] != 'macosx_*' }} | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: sdist | |
| path: ./dist | |
| - name: Output sdist name (macOS) | |
| id: save-path | |
| shell: bash -el {0} | |
| run: echo "sdist_name=$(ls ./dist)" >> "$GITHUB_ENV" | |
| # Python version used to build sdist doesn't matter | |
| # wheel will be built from sdist with the correct version | |
| - name: Unzip sdist (macOS) | |
| if: ${{ startsWith(matrix.buildplat[1], 'macosx') }} | |
| run: | | |
| tar -xzf ./dist/${{ env.sdist_name }} -C ./dist | |
| - name: Output sdist name (macOS) | |
| id: save-path2 | |
| shell: bash -el {0} | |
| run: echo "sdist_name=$(cd ./dist && ls -d */)" >> "$GITHUB_ENV" | |
| - name: Build wheels | |
| uses: pypa/[email protected] | |
| with: | |
| package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }} | |
| env: | |
| CIBW_PRERELEASE_PYTHONS: True | |
| CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }} | |
| CIBW_PLATFORM: ${{ matrix.buildplat[1] == 'pyodide_wasm32' && 'pyodide' || 'auto' }} | |
| - name: Set up Python | |
| uses: mamba-org/setup-micromamba@v1 | |
| with: | |
| environment-name: wheel-env | |
| # Use a fixed Python, since we might have an unreleased Python not | |
| # yet present on conda-forge | |
| create-args: >- | |
| python=3.11 | |
| anaconda-client | |
| wheel | |
| cache-downloads: true | |
| cache-environment: true | |
| - name: Validate wheel RECORD | |
| shell: bash -el {0} | |
| run: for whl in $(ls wheelhouse); do wheel unpack wheelhouse/$whl -d /tmp; done | |
| # Testing on windowsservercore instead of GHA runner to fail on missing DLLs | |
| - name: Test Windows Wheels | |
| if: ${{ matrix.buildplat[1] == 'win_amd64' }} | |
| shell: pwsh | |
| run: | | |
| $TST_CMD = @" | |
| python -m pip install hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0; | |
| python -m pip install `$(Get-Item pandas\wheelhouse\*.whl); | |
| python -c `'import pandas as pd; pd.test(extra_args=[`\"--no-strict-data-files`\", `\"-m not clipboard and not single_cpu and not slow and not network and not db`\"])`'; | |
| "@ | |
| # add rc to the end of the image name if the Python version is unreleased | |
| docker pull python:${{ matrix.python[1] == '3.12' && '3.12-rc' || format('{0}-windowsservercore', matrix.python[1]) }} | |
| docker run --env PANDAS_CI='1' -v ${PWD}:C:\pandas python:${{ matrix.python[1] == '3.12' && '3.12-rc' || format('{0}-windowsservercore', matrix.python[1]) }} powershell -Command $TST_CMD | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }} | |
| path: ./wheelhouse/*.whl | |
| - name: Upload wheels & sdist | |
| if: ${{ success() && (env.IS_SCHEDULE_DISPATCH == 'true' || env.IS_PUSH == 'true') }} | |
| shell: bash -el {0} | |
| env: | |
| PANDAS_STAGING_UPLOAD_TOKEN: ${{ secrets.PANDAS_STAGING_UPLOAD_TOKEN }} | |
| PANDAS_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.PANDAS_NIGHTLY_UPLOAD_TOKEN }} | |
| # trigger an upload to | |
| # https://anaconda.org/scientific-python-nightly-wheels/pandas | |
| # for cron jobs or "Run workflow" (restricted to main branch). | |
| # Tags will upload to | |
| # https://anaconda.org/multibuild-wheels-staging/pandas | |
| # The tokens were originally generated at anaconda.org | |
| run: | | |
| source ci/upload_wheels.sh | |
| set_upload_vars | |
| upload_wheels |