release.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592
  1. #!/usr/bin/env python
  2. # -*- coding: utf-8 -*-
  3. # Copyright 2020 The Matrix.org Foundation C.I.C.
  4. #
  5. # Licensed under the Apache License, Version 2.0 (the "License");
  6. # you may not use this file except in compliance with the License.
  7. # You may obtain a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS,
  13. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. """An interactive script for doing a release. See `cli()` below.
  17. """
  18. import glob
  19. import os
  20. import re
  21. import subprocess
  22. import sys
  23. import urllib.request
  24. from os import path
  25. from tempfile import TemporaryDirectory
  26. from typing import List, Optional, Tuple
  27. import attr
  28. import click
  29. import commonmark
  30. import git
  31. import redbaron
  32. from click.exceptions import ClickException
  33. from github import Github
  34. from packaging import version
  35. def run_until_successful(command, *args, **kwargs):
  36. while True:
  37. completed_process = subprocess.run(command, *args, **kwargs)
  38. exit_code = completed_process.returncode
  39. if exit_code == 0:
  40. # successful, so nothing more to do here.
  41. return completed_process
  42. print(f"The command {command!r} failed with exit code {exit_code}.")
  43. print("Please try to correct the failure and then re-run.")
  44. click.confirm("Try again?", abort=True)
  45. @click.group()
  46. def cli():
  47. """An interactive script to walk through the parts of creating a release.
  48. Requires the dev dependencies be installed, which can be done via:
  49. pip install -e .[dev]
  50. Then to use:
  51. ./scripts-dev/release.py prepare
  52. # ... ask others to look at the changelog ...
  53. ./scripts-dev/release.py tag
  54. # ... wait for assets to build ...
  55. ./scripts-dev/release.py publish
  56. ./scripts-dev/release.py upload
  57. # Optional: generate some nice links for the announcement
  58. ./scripts-dev/release.py upload
  59. If the env var GH_TOKEN (or GITHUB_TOKEN) is set, or passed into the
  60. `tag`/`publish` command, then a new draft release will be created/published.
  61. """
  62. @cli.command()
  63. def prepare():
  64. """Do the initial stages of creating a release, including creating release
  65. branch, updating changelog and pushing to GitHub.
  66. """
  67. # Make sure we're in a git repo.
  68. try:
  69. repo = git.Repo()
  70. except git.InvalidGitRepositoryError:
  71. raise click.ClickException("Not in Synapse repo.")
  72. if repo.is_dirty():
  73. raise click.ClickException("Uncommitted changes exist.")
  74. click.secho("Updating git repo...")
  75. repo.remote().fetch()
  76. # Get the current version and AST from root Synapse module.
  77. current_version, parsed_synapse_ast, version_node = parse_version_from_module()
  78. # Figure out what sort of release we're doing and calcuate the new version.
  79. rc = click.confirm("RC", default=True)
  80. if current_version.pre:
  81. # If the current version is an RC we don't need to bump any of the
  82. # version numbers (other than the RC number).
  83. if rc:
  84. new_version = "{}.{}.{}rc{}".format(
  85. current_version.major,
  86. current_version.minor,
  87. current_version.micro,
  88. current_version.pre[1] + 1,
  89. )
  90. else:
  91. new_version = "{}.{}.{}".format(
  92. current_version.major,
  93. current_version.minor,
  94. current_version.micro,
  95. )
  96. else:
  97. # If this is a new release cycle then we need to know if it's a minor
  98. # or a patch version bump.
  99. release_type = click.prompt(
  100. "Release type",
  101. type=click.Choice(("minor", "patch")),
  102. show_choices=True,
  103. default="minor",
  104. )
  105. if release_type == "minor":
  106. if rc:
  107. new_version = "{}.{}.{}rc1".format(
  108. current_version.major,
  109. current_version.minor + 1,
  110. 0,
  111. )
  112. else:
  113. new_version = "{}.{}.{}".format(
  114. current_version.major,
  115. current_version.minor + 1,
  116. 0,
  117. )
  118. else:
  119. if rc:
  120. new_version = "{}.{}.{}rc1".format(
  121. current_version.major,
  122. current_version.minor,
  123. current_version.micro + 1,
  124. )
  125. else:
  126. new_version = "{}.{}.{}".format(
  127. current_version.major,
  128. current_version.minor,
  129. current_version.micro + 1,
  130. )
  131. # Confirm the calculated version is OK.
  132. if not click.confirm(f"Create new version: {new_version}?", default=True):
  133. click.get_current_context().abort()
  134. # Switch to the release branch.
  135. parsed_new_version = version.parse(new_version)
  136. # We assume for debian changelogs that we only do RCs or full releases.
  137. assert not parsed_new_version.is_devrelease
  138. assert not parsed_new_version.is_postrelease
  139. release_branch_name = (
  140. f"release-v{parsed_new_version.major}.{parsed_new_version.minor}"
  141. )
  142. release_branch = find_ref(repo, release_branch_name)
  143. if release_branch:
  144. if release_branch.is_remote():
  145. # If the release branch only exists on the remote we check it out
  146. # locally.
  147. repo.git.checkout(release_branch_name)
  148. release_branch = repo.active_branch
  149. else:
  150. # If a branch doesn't exist we create one. We ask which one branch it
  151. # should be based off, defaulting to sensible values depending on the
  152. # release type.
  153. if current_version.is_prerelease:
  154. default = release_branch_name
  155. elif release_type == "minor":
  156. default = "develop"
  157. else:
  158. default = "master"
  159. branch_name = click.prompt(
  160. "Which branch should the release be based on?", default=default
  161. )
  162. base_branch = find_ref(repo, branch_name)
  163. if not base_branch:
  164. print(f"Could not find base branch {branch_name}!")
  165. click.get_current_context().abort()
  166. # Check out the base branch and ensure it's up to date
  167. repo.head.reference = base_branch
  168. repo.head.reset(index=True, working_tree=True)
  169. if not base_branch.is_remote():
  170. update_branch(repo)
  171. # Create the new release branch
  172. release_branch = repo.create_head(release_branch_name, commit=base_branch)
  173. # Switch to the release branch and ensure its up to date.
  174. repo.git.checkout(release_branch_name)
  175. update_branch(repo)
  176. # Update the `__version__` variable and write it back to the file.
  177. version_node.value = '"' + new_version + '"'
  178. with open("synapse/__init__.py", "w") as f:
  179. f.write(parsed_synapse_ast.dumps())
  180. # Generate changelogs.
  181. generate_and_write_changelog(current_version)
  182. # Generate debian changelogs
  183. if parsed_new_version.pre is not None:
  184. # If this is an RC then we need to coerce the version string to match
  185. # Debian norms, e.g. 1.39.0rc2 gets converted to 1.39.0~rc2.
  186. base_ver = parsed_new_version.base_version
  187. pre_type, pre_num = parsed_new_version.pre
  188. debian_version = f"{base_ver}~{pre_type}{pre_num}"
  189. else:
  190. debian_version = new_version
  191. run_until_successful(
  192. f'dch -M -v {debian_version} "New synapse release {debian_version}."',
  193. shell=True,
  194. )
  195. run_until_successful('dch -M -r -D stable ""', shell=True)
  196. # Show the user the changes and ask if they want to edit the change log.
  197. repo.git.add("-u")
  198. subprocess.run("git diff --cached", shell=True)
  199. if click.confirm("Edit changelog?", default=False):
  200. click.edit(filename="CHANGES.md")
  201. # Commit the changes.
  202. repo.git.add("-u")
  203. repo.git.commit("-m", new_version)
  204. # We give the option to bail here in case the user wants to make sure things
  205. # are OK before pushing.
  206. if not click.confirm("Push branch to github?", default=True):
  207. print("")
  208. print("Run when ready to push:")
  209. print("")
  210. print(f"\tgit push -u {repo.remote().name} {repo.active_branch.name}")
  211. print("")
  212. sys.exit(0)
  213. # Otherwise, push and open the changelog in the browser.
  214. repo.git.push("-u", repo.remote().name, repo.active_branch.name)
  215. print("Opening the changelog in your browser...")
  216. print("Please ask others to give it a check.")
  217. click.launch(
  218. f"https://github.com/matrix-org/synapse/blob/{repo.active_branch.name}/CHANGES.md"
  219. )
  220. @cli.command()
  221. @click.option("--gh-token", envvar=["GH_TOKEN", "GITHUB_TOKEN"])
  222. def tag(gh_token: Optional[str]):
  223. """Tags the release and generates a draft GitHub release"""
  224. # Make sure we're in a git repo.
  225. try:
  226. repo = git.Repo()
  227. except git.InvalidGitRepositoryError:
  228. raise click.ClickException("Not in Synapse repo.")
  229. if repo.is_dirty():
  230. raise click.ClickException("Uncommitted changes exist.")
  231. click.secho("Updating git repo...")
  232. repo.remote().fetch()
  233. # Find out the version and tag name.
  234. current_version, _, _ = parse_version_from_module()
  235. tag_name = f"v{current_version}"
  236. # Check we haven't released this version.
  237. if tag_name in repo.tags:
  238. raise click.ClickException(f"Tag {tag_name} already exists!\n")
  239. # Get the appropriate changelogs and tag.
  240. changes = get_changes_for_version(current_version)
  241. click.echo_via_pager(changes)
  242. if click.confirm("Edit text?", default=False):
  243. changes = click.edit(changes, require_save=False)
  244. repo.create_tag(tag_name, message=changes, sign=True)
  245. if not click.confirm("Push tag to GitHub?", default=True):
  246. print("")
  247. print("Run when ready to push:")
  248. print("")
  249. print(f"\tgit push {repo.remote().name} tag {current_version}")
  250. print("")
  251. return
  252. repo.git.push(repo.remote().name, "tag", tag_name)
  253. # If no token was given, we bail here
  254. if not gh_token:
  255. print("Launching the GitHub release page in your browser.")
  256. print("Please correct the title and create a draft.")
  257. if current_version.is_prerelease:
  258. print("As this is an RC, remember to mark it as a pre-release!")
  259. print("(by the way, this step can be automated by passing --gh-token,")
  260. print("or one of the GH_TOKEN or GITHUB_TOKEN env vars.)")
  261. click.launch(f"https://github.com/matrix-org/synapse/releases/edit/{tag_name}")
  262. print("Once done, you need to wait for the release assets to build.")
  263. if click.confirm("Launch the release assets actions page?", default=True):
  264. click.launch(
  265. f"https://github.com/matrix-org/synapse/actions?query=branch%3A{tag_name}"
  266. )
  267. return
  268. # Create a new draft release
  269. gh = Github(gh_token)
  270. gh_repo = gh.get_repo("matrix-org/synapse")
  271. release = gh_repo.create_git_release(
  272. tag=tag_name,
  273. name=tag_name,
  274. message=changes,
  275. draft=True,
  276. prerelease=current_version.is_prerelease,
  277. )
  278. # Open the release and the actions where we are building the assets.
  279. print("Launching the release page and the actions page.")
  280. click.launch(release.html_url)
  281. click.launch(
  282. f"https://github.com/matrix-org/synapse/actions?query=branch%3A{tag_name}"
  283. )
  284. click.echo("Wait for release assets to be built")
  285. @cli.command()
  286. @click.option("--gh-token", envvar=["GH_TOKEN", "GITHUB_TOKEN"], required=True)
  287. def publish(gh_token: str):
  288. """Publish release."""
  289. # Make sure we're in a git repo.
  290. try:
  291. repo = git.Repo()
  292. except git.InvalidGitRepositoryError:
  293. raise click.ClickException("Not in Synapse repo.")
  294. if repo.is_dirty():
  295. raise click.ClickException("Uncommitted changes exist.")
  296. current_version, _, _ = parse_version_from_module()
  297. tag_name = f"v{current_version}"
  298. if not click.confirm(f"Publish {tag_name}?", default=True):
  299. return
  300. # Publish the draft release
  301. gh = Github(gh_token)
  302. gh_repo = gh.get_repo("matrix-org/synapse")
  303. for release in gh_repo.get_releases():
  304. if release.title == tag_name:
  305. break
  306. else:
  307. raise ClickException(f"Failed to find GitHub release for {tag_name}")
  308. assert release.title == tag_name
  309. if not release.draft:
  310. click.echo("Release already published.")
  311. return
  312. release = release.update_release(
  313. name=release.title,
  314. message=release.body,
  315. tag_name=release.tag_name,
  316. prerelease=release.prerelease,
  317. draft=False,
  318. )
  319. @cli.command()
  320. def upload():
  321. """Upload release to pypi."""
  322. current_version, _, _ = parse_version_from_module()
  323. tag_name = f"v{current_version}"
  324. pypi_asset_names = [
  325. f"matrix_synapse-{current_version}-py3-none-any.whl",
  326. f"matrix-synapse-{current_version}.tar.gz",
  327. ]
  328. with TemporaryDirectory(prefix=f"synapse_upload_{tag_name}_") as tmpdir:
  329. for name in pypi_asset_names:
  330. filename = path.join(tmpdir, name)
  331. url = f"https://github.com/matrix-org/synapse/releases/download/{tag_name}/{name}"
  332. click.echo(f"Downloading {name} into {filename}")
  333. urllib.request.urlretrieve(url, filename=filename)
  334. if click.confirm("Upload to PyPI?", default=True):
  335. subprocess.run("twine upload *", shell=True, cwd=tmpdir)
  336. click.echo(
  337. f"Done! Remember to merge the tag {tag_name} into the appropriate branches"
  338. )
  339. @cli.command()
  340. def announce():
  341. """Generate markdown to announce the release."""
  342. current_version, _, _ = parse_version_from_module()
  343. tag_name = f"v{current_version}"
  344. click.echo(
  345. f"""
  346. Hi everyone. Synapse {current_version} has just been released.
  347. [notes](https://github.com/matrix-org/synapse/releases/tag/{tag_name}) |\
  348. [docker](https://hub.docker.com/r/matrixdotorg/synapse/tags?name={tag_name}) | \
  349. [debs](https://packages.matrix.org/debian/) | \
  350. [pypi](https://pypi.org/project/matrix-synapse/{current_version}/)"""
  351. )
  352. if "rc" in tag_name:
  353. click.echo(
  354. """
  355. Announce the RC in
  356. - #homeowners:matrix.org (Synapse Announcements)
  357. - #synapse-dev:matrix.org"""
  358. )
  359. else:
  360. click.echo(
  361. """
  362. Announce the release in
  363. - #homeowners:matrix.org (Synapse Announcements), bumping the version in the topic
  364. - #synapse:matrix.org (Synapse Admins), bumping the version in the topic
  365. - #synapse-dev:matrix.org
  366. - #synapse-package-maintainers:matrix.org"""
  367. )
  368. def parse_version_from_module() -> Tuple[
  369. version.Version, redbaron.RedBaron, redbaron.Node
  370. ]:
  371. # Parse the AST and load the `__version__` node so that we can edit it
  372. # later.
  373. with open("synapse/__init__.py") as f:
  374. red = redbaron.RedBaron(f.read())
  375. version_node = None
  376. for node in red:
  377. if node.type != "assignment":
  378. continue
  379. if node.target.type != "name":
  380. continue
  381. if node.target.value != "__version__":
  382. continue
  383. version_node = node
  384. break
  385. if not version_node:
  386. print("Failed to find '__version__' definition in synapse/__init__.py")
  387. sys.exit(1)
  388. # Parse the current version.
  389. current_version = version.parse(version_node.value.value.strip('"'))
  390. assert isinstance(current_version, version.Version)
  391. return current_version, red, version_node
  392. def find_ref(repo: git.Repo, ref_name: str) -> Optional[git.HEAD]:
  393. """Find the branch/ref, looking first locally then in the remote."""
  394. if ref_name in repo.refs:
  395. return repo.refs[ref_name]
  396. elif ref_name in repo.remote().refs:
  397. return repo.remote().refs[ref_name]
  398. else:
  399. return None
  400. def update_branch(repo: git.Repo):
  401. """Ensure branch is up to date if it has a remote"""
  402. if repo.active_branch.tracking_branch():
  403. repo.git.merge(repo.active_branch.tracking_branch().name)
  404. def get_changes_for_version(wanted_version: version.Version) -> str:
  405. """Get the changelogs for the given version.
  406. If an RC then will only get the changelog for that RC version, otherwise if
  407. its a full release will get the changelog for the release and all its RCs.
  408. """
  409. with open("CHANGES.md") as f:
  410. changes = f.read()
  411. # First we parse the changelog so that we can split it into sections based
  412. # on the release headings.
  413. ast = commonmark.Parser().parse(changes)
  414. @attr.s(auto_attribs=True)
  415. class VersionSection:
  416. title: str
  417. # These are 0-based.
  418. start_line: int
  419. end_line: Optional[int] = None # Is none if its the last entry
  420. headings: List[VersionSection] = []
  421. for node, _ in ast.walker():
  422. # We look for all text nodes that are in a level 1 heading.
  423. if node.t != "text":
  424. continue
  425. if node.parent.t != "heading" or node.parent.level != 1:
  426. continue
  427. # If we have a previous heading then we update its `end_line`.
  428. if headings:
  429. headings[-1].end_line = node.parent.sourcepos[0][0] - 1
  430. headings.append(VersionSection(node.literal, node.parent.sourcepos[0][0] - 1))
  431. changes_by_line = changes.split("\n")
  432. version_changelog = [] # The lines we want to include in the changelog
  433. # Go through each section and find any that match the requested version.
  434. regex = re.compile(r"^Synapse v?(\S+)")
  435. for section in headings:
  436. groups = regex.match(section.title)
  437. if not groups:
  438. continue
  439. heading_version = version.parse(groups.group(1))
  440. heading_base_version = version.parse(heading_version.base_version)
  441. # Check if heading version matches the requested version, or if its an
  442. # RC of the requested version.
  443. if wanted_version not in (heading_version, heading_base_version):
  444. continue
  445. version_changelog.extend(changes_by_line[section.start_line : section.end_line])
  446. return "\n".join(version_changelog)
  447. def generate_and_write_changelog(current_version: version.Version):
  448. # We do this by getting a draft so that we can edit it before writing to the
  449. # changelog.
  450. result = run_until_successful(
  451. "python3 -m towncrier --draft", shell=True, capture_output=True
  452. )
  453. new_changes = result.stdout.decode("utf-8")
  454. new_changes = new_changes.replace(
  455. "No significant changes.", f"No significant changes since {current_version}."
  456. )
  457. # Prepend changes to changelog
  458. with open("CHANGES.md", "r+") as f:
  459. existing_content = f.read()
  460. f.seek(0, 0)
  461. f.write(new_changes)
  462. f.write("\n")
  463. f.write(existing_content)
  464. # Remove all the news fragments
  465. for f in glob.iglob("changelog.d/*.*"):
  466. os.remove(f)
  467. if __name__ == "__main__":
  468. cli()