latest_deps.yml 8.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234
  1. # People who are freshly `pip install`ing from PyPI will pull in the latest versions of
  2. # dependencies which match the broad requirements. Since most CI runs are against
  3. # the locked poetry environment, run specifically against the latest dependencies to
  4. # know if there's an upcoming breaking change.
  5. #
  6. # As an overview this workflow:
  7. # - checks out develop,
  8. # - installs from source, pulling in the dependencies like a fresh `pip install` would, and
  9. # - runs mypy and test suites in that checkout.
  10. #
  11. # Based on the twisted trunk CI job.
  12. name: Latest dependencies
  13. on:
  14. schedule:
  15. - cron: 0 7 * * *
  16. workflow_dispatch:
  17. concurrency:
  18. group: ${{ github.workflow }}-${{ github.ref }}
  19. cancel-in-progress: true
  20. jobs:
  21. check_repo:
  22. # Prevent this workflow from running on any fork of Synapse other than matrix-org/synapse, as it is
  23. # only useful to the Synapse core team.
  24. # All other workflow steps depend on this one, thus if 'should_run_workflow' is not 'true', the rest
  25. # of the workflow will be skipped as well.
  26. runs-on: ubuntu-latest
  27. outputs:
  28. should_run_workflow: ${{ steps.check_condition.outputs.should_run_workflow }}
  29. steps:
  30. - id: check_condition
  31. run: echo "should_run_workflow=${{ github.repository == 'matrix-org/synapse' }}" >> "$GITHUB_OUTPUT"
  32. mypy:
  33. needs: check_repo
  34. if: needs.check_repo.outputs.should_run_workflow == 'true'
  35. runs-on: ubuntu-latest
  36. steps:
  37. - uses: actions/checkout@v4
  38. - name: Install Rust
  39. uses: dtolnay/rust-toolchain@stable
  40. - uses: Swatinem/rust-cache@v2
  41. # The dev dependencies aren't exposed in the wheel metadata (at least with current
  42. # poetry-core versions), so we install with poetry.
  43. - uses: matrix-org/setup-python-poetry@v1
  44. with:
  45. python-version: "3.x"
  46. poetry-version: "1.3.2"
  47. extras: "all"
  48. # Dump installed versions for debugging.
  49. - run: poetry run pip list > before.txt
  50. # Upgrade all runtime dependencies only. This is intended to mimic a fresh
  51. # `pip install matrix-synapse[all]` as closely as possible.
  52. - run: poetry update --no-dev
  53. - run: poetry run pip list > after.txt && (diff -u before.txt after.txt || true)
  54. - name: Remove unhelpful options from mypy config
  55. run: sed -e '/warn_unused_ignores = True/d' -e '/warn_redundant_casts = True/d' -i mypy.ini
  56. - run: poetry run mypy
  57. trial:
  58. needs: check_repo
  59. if: needs.check_repo.outputs.should_run_workflow == 'true'
  60. runs-on: ubuntu-latest
  61. strategy:
  62. matrix:
  63. include:
  64. - database: "sqlite"
  65. - database: "postgres"
  66. postgres-version: "14"
  67. steps:
  68. - uses: actions/checkout@v4
  69. - name: Install Rust
  70. uses: dtolnay/rust-toolchain@stable
  71. - uses: Swatinem/rust-cache@v2
  72. - run: sudo apt-get -qq install xmlsec1
  73. - name: Set up PostgreSQL ${{ matrix.postgres-version }}
  74. if: ${{ matrix.postgres-version }}
  75. run: |
  76. docker run -d -p 5432:5432 \
  77. -e POSTGRES_PASSWORD=postgres \
  78. -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
  79. postgres:${{ matrix.postgres-version }}
  80. - uses: actions/setup-python@v4
  81. with:
  82. python-version: "3.x"
  83. - run: pip install .[all,test]
  84. - name: Await PostgreSQL
  85. if: ${{ matrix.postgres-version }}
  86. timeout-minutes: 2
  87. run: until pg_isready -h localhost; do sleep 1; done
  88. # We nuke the local copy, as we've installed synapse into the virtualenv
  89. # (rather than use an editable install, which we no longer support). If we
  90. # don't do this then python can't find the native lib.
  91. - run: rm -rf synapse/
  92. - run: python -m twisted.trial --jobs=2 tests
  93. env:
  94. SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
  95. SYNAPSE_POSTGRES_HOST: localhost
  96. SYNAPSE_POSTGRES_USER: postgres
  97. SYNAPSE_POSTGRES_PASSWORD: postgres
  98. - name: Dump logs
  99. # Logs are most useful when the command fails, always include them.
  100. if: ${{ always() }}
  101. # Note: Dumps to workflow logs instead of using actions/upload-artifact
  102. # This keeps logs colocated with failing jobs
  103. # It also ignores find's exit code; this is a best effort affair
  104. run: >-
  105. find _trial_temp -name '*.log'
  106. -exec echo "::group::{}" \;
  107. -exec cat {} \;
  108. -exec echo "::endgroup::" \;
  109. || true
  110. sytest:
  111. needs: check_repo
  112. if: needs.check_repo.outputs.should_run_workflow == 'true'
  113. runs-on: ubuntu-latest
  114. container:
  115. image: matrixdotorg/sytest-synapse:testing
  116. volumes:
  117. - ${{ github.workspace }}:/src
  118. strategy:
  119. fail-fast: false
  120. matrix:
  121. include:
  122. - sytest-tag: focal
  123. - sytest-tag: focal
  124. postgres: postgres
  125. workers: workers
  126. redis: redis
  127. env:
  128. POSTGRES: ${{ matrix.postgres && 1}}
  129. WORKERS: ${{ matrix.workers && 1 }}
  130. REDIS: ${{ matrix.redis && 1 }}
  131. BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
  132. steps:
  133. - uses: actions/checkout@v4
  134. - name: Install Rust
  135. uses: dtolnay/rust-toolchain@stable
  136. - uses: Swatinem/rust-cache@v2
  137. - name: Ensure sytest runs `pip install`
  138. # Delete the lockfile so sytest will `pip install` rather than `poetry install`
  139. run: rm /src/poetry.lock
  140. working-directory: /src
  141. - name: Prepare test blacklist
  142. run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
  143. - name: Run SyTest
  144. run: /bootstrap.sh synapse
  145. working-directory: /src
  146. - name: Summarise results.tap
  147. if: ${{ always() }}
  148. run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
  149. - name: Upload SyTest logs
  150. uses: actions/upload-artifact@v3
  151. if: ${{ always() }}
  152. with:
  153. name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
  154. path: |
  155. /logs/results.tap
  156. /logs/**/*.log*
  157. complement:
  158. needs: check_repo
  159. if: "!failure() && !cancelled() && needs.check_repo.outputs.should_run_workflow == 'true'"
  160. runs-on: ubuntu-latest
  161. strategy:
  162. fail-fast: false
  163. matrix:
  164. include:
  165. - arrangement: monolith
  166. database: SQLite
  167. - arrangement: monolith
  168. database: Postgres
  169. - arrangement: workers
  170. database: Postgres
  171. steps:
  172. - name: Run actions/checkout@v4 for synapse
  173. uses: actions/checkout@v4
  174. with:
  175. path: synapse
  176. - name: Prepare Complement's Prerequisites
  177. run: synapse/.ci/scripts/setup_complement_prerequisites.sh
  178. - uses: actions/setup-go@v4
  179. with:
  180. cache-dependency-path: complement/go.sum
  181. go-version-file: complement/go.mod
  182. - run: |
  183. set -o pipefail
  184. TEST_ONLY_IGNORE_POETRY_LOCKFILE=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt
  185. shell: bash
  186. name: Run Complement Tests
  187. # Open an issue if the build fails, so we know about it.
  188. # Only do this if we're not experimenting with this action in a PR.
  189. open-issue:
  190. if: "failure() && github.event_name != 'push' && github.event_name != 'pull_request' && needs.check_repo.outputs.should_run_workflow == 'true'"
  191. needs:
  192. # TODO: should mypy be included here? It feels more brittle than the others.
  193. - mypy
  194. - trial
  195. - sytest
  196. - complement
  197. runs-on: ubuntu-latest
  198. steps:
  199. - uses: actions/checkout@v4
  200. - uses: JasonEtco/create-an-issue@e27dddc79c92bc6e4562f268fffa5ed752639abd # v2.9.1
  201. env:
  202. GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  203. with:
  204. update_existing: true
  205. filename: .ci/latest_deps_build_failed_issue_template.md