release-artifacts.yml 6.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212
  1. # GitHub actions workflow which builds the release artifacts.
  2. name: Build release artifacts
  3. on:
  4. # we build on PRs and develop to (hopefully) get early warning
  5. # of things breaking (but only build one set of debs). PRs skip
  6. # building wheels on macOS & ARM.
  7. pull_request:
  8. push:
  9. branches: ["develop", "release-*"]
  10. # we do the full build on tags.
  11. tags: ["v*"]
  12. merge_group:
  13. workflow_dispatch:
  14. concurrency:
  15. group: ${{ github.workflow }}-${{ github.ref }}
  16. cancel-in-progress: true
  17. permissions:
  18. contents: write
  19. jobs:
  20. get-distros:
  21. name: "Calculate list of debian distros"
  22. runs-on: ubuntu-latest
  23. steps:
  24. - uses: actions/checkout@v3
  25. - uses: actions/setup-python@v4
  26. with:
  27. python-version: '3.x'
  28. - id: set-distros
  29. run: |
  30. # if we're running from a tag, get the full list of distros; otherwise just use debian:sid
  31. # NOTE: inside the actual Dockerfile-dhvirtualenv, the image name is expanded into its full image path
  32. dists='["debian:sid"]'
  33. if [[ $GITHUB_REF == refs/tags/* ]]; then
  34. dists=$(scripts-dev/build_debian_packages.py --show-dists-json)
  35. fi
  36. echo "distros=$dists" >> "$GITHUB_OUTPUT"
  37. # map the step outputs to job outputs
  38. outputs:
  39. distros: ${{ steps.set-distros.outputs.distros }}
  40. # now build the packages with a matrix build.
  41. build-debs:
  42. needs: get-distros
  43. name: "Build .deb packages"
  44. runs-on: ubuntu-latest
  45. strategy:
  46. matrix:
  47. distro: ${{ fromJson(needs.get-distros.outputs.distros) }}
  48. steps:
  49. - name: Checkout
  50. uses: actions/checkout@v3
  51. with:
  52. path: src
  53. - name: Set up Docker Buildx
  54. id: buildx
  55. uses: docker/setup-buildx-action@v2
  56. with:
  57. install: true
  58. - name: Set up docker layer caching
  59. uses: actions/cache@v3
  60. with:
  61. path: /tmp/.buildx-cache
  62. key: ${{ runner.os }}-buildx-${{ github.sha }}
  63. restore-keys: |
  64. ${{ runner.os }}-buildx-
  65. - name: Set up python
  66. uses: actions/setup-python@v4
  67. with:
  68. python-version: '3.x'
  69. - name: Build the packages
  70. # see https://github.com/docker/build-push-action/issues/252
  71. # for the cache magic here
  72. run: |
  73. ./src/scripts-dev/build_debian_packages.py \
  74. --docker-build-arg=--cache-from=type=local,src=/tmp/.buildx-cache \
  75. --docker-build-arg=--cache-to=type=local,mode=max,dest=/tmp/.buildx-cache-new \
  76. --docker-build-arg=--progress=plain \
  77. --docker-build-arg=--load \
  78. "${{ matrix.distro }}"
  79. rm -rf /tmp/.buildx-cache
  80. mv /tmp/.buildx-cache-new /tmp/.buildx-cache
  81. - name: Upload debs as artifacts
  82. uses: actions/upload-artifact@v3
  83. with:
  84. name: debs
  85. path: debs/*
  86. build-wheels:
  87. name: Build wheels on ${{ matrix.os }} for ${{ matrix.arch }}
  88. runs-on: ${{ matrix.os }}
  89. strategy:
  90. matrix:
  91. os: [ubuntu-20.04, macos-11]
  92. arch: [x86_64, aarch64]
  93. # is_pr is a flag used to exclude certain jobs from the matrix on PRs.
  94. # It is not read by the rest of the workflow.
  95. is_pr:
  96. - ${{ startsWith(github.ref, 'refs/pull/') }}
  97. exclude:
  98. # Don't build macos wheels on PR CI.
  99. - is_pr: true
  100. os: "macos-11"
  101. # Don't build aarch64 wheels on mac.
  102. - os: "macos-11"
  103. arch: aarch64
  104. # Don't build aarch64 wheels on PR CI.
  105. - is_pr: true
  106. arch: aarch64
  107. steps:
  108. - uses: actions/checkout@v3
  109. - uses: actions/setup-python@v4
  110. with:
  111. # setup-python@v4 doesn't impose a default python version. Need to use 3.x
  112. # here, because `python` on osx points to Python 2.7.
  113. python-version: "3.x"
  114. - name: Install cibuildwheel
  115. run: python -m pip install cibuildwheel==2.9.0
  116. - name: Set up QEMU to emulate aarch64
  117. if: matrix.arch == 'aarch64'
  118. uses: docker/setup-qemu-action@v2
  119. with:
  120. platforms: arm64
  121. - name: Build aarch64 wheels
  122. if: matrix.arch == 'aarch64'
  123. run: echo 'CIBW_ARCHS_LINUX=aarch64' >> $GITHUB_ENV
  124. - name: Only build a single wheel on PR
  125. if: startsWith(github.ref, 'refs/pull/')
  126. run: echo "CIBW_BUILD="cp38-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV
  127. - name: Build wheels
  128. run: python -m cibuildwheel --output-dir wheelhouse
  129. env:
  130. # Skip testing for platforms which various libraries don't have wheels
  131. # for, and so need extra build deps.
  132. CIBW_TEST_SKIP: pp3*-* *i686* *musl*
  133. # Fix Rust OOM errors on emulated aarch64: https://github.com/rust-lang/cargo/issues/10583
  134. CARGO_NET_GIT_FETCH_WITH_CLI: true
  135. CIBW_ENVIRONMENT_PASS_LINUX: CARGO_NET_GIT_FETCH_WITH_CLI
  136. - uses: actions/upload-artifact@v3
  137. with:
  138. name: Wheel
  139. path: ./wheelhouse/*.whl
  140. build-sdist:
  141. name: Build sdist
  142. runs-on: ubuntu-latest
  143. if: ${{ !startsWith(github.ref, 'refs/pull/') }}
  144. steps:
  145. - uses: actions/checkout@v3
  146. - uses: actions/setup-python@v4
  147. with:
  148. python-version: '3.10'
  149. - run: pip install build
  150. - name: Build sdist
  151. run: python -m build --sdist
  152. - uses: actions/upload-artifact@v3
  153. with:
  154. name: Sdist
  155. path: dist/*.tar.gz
  156. # if it's a tag, create a release and attach the artifacts to it
  157. attach-assets:
  158. name: "Attach assets to release"
  159. if: ${{ !failure() && !cancelled() && startsWith(github.ref, 'refs/tags/') }}
  160. needs:
  161. - build-debs
  162. - build-wheels
  163. - build-sdist
  164. runs-on: ubuntu-latest
  165. steps:
  166. - name: Download all workflow run artifacts
  167. uses: actions/download-artifact@v3
  168. - name: Build a tarball for the debs
  169. run: tar -cvJf debs.tar.xz debs
  170. - name: Attach to release
  171. uses: softprops/action-gh-release@a929a66f232c1b11af63782948aa2210f981808a # PR#109
  172. env:
  173. GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  174. with:
  175. files: |
  176. Sdist/*
  177. Wheel/*
  178. debs.tar.xz
  179. # if it's not already published, keep the release as a draft.
  180. draft: true
  181. # mark it as a prerelease if the tag contains 'rc'.
  182. prerelease: ${{ contains(github.ref, 'rc') }}