From 8a86f38bca61ab67dcc137c4966f32b3dc2251d9 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Wed, 7 Aug 2024 20:14:45 -0700 Subject: [PATCH 1/3] Fix is_maintenance_branch heuristic This was broken because Nix language's version comparison does not know how to deal with versions like -rc1 and considers them newer, which is in this case not desirable. That in turn led to not tagging 2.90.0 docker images as "latest" since the heuristic was wrong. This commit also adds some more cross-checking and failsafes in case the person running releng does not have a local main branch that is up to date. Fixes: https://git.lix.systems/lix-project/lix/issues/443 Change-Id: I537103ebab58ae978c00e06972abe14432dd9c80 --- package.nix | 1 + releng/gitutils.xsh | 37 ++++++++++++++++++++++++++----------- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/package.nix b/package.nix index be2f0010d..c32539303 100644 --- a/package.nix +++ b/package.nix @@ -423,6 +423,7 @@ stdenv.mkDerivation (finalAttrs: { p.python-frontmatter p.requests p.xdg-base-dirs + p.packaging (p.toPythonModule xonsh.passthru.unwrapped) ] ); diff --git a/releng/gitutils.xsh b/releng/gitutils.xsh index e18b4da5f..d3b3774c0 100644 --- a/releng/gitutils.xsh +++ b/releng/gitutils.xsh @@ -1,11 +1,24 @@ import subprocess -import json +from packaging.version import Version from .version import VERSION +def remote_is_plausible(url: str) -> bool: + return ('git.lix.systems' in url and 'lix-project/lix' in url) or ('gerrit.lix.systems' in url and url.endswith('lix')) + + def version_compare(v1: str, v2: str): - return json.loads($(nix-instantiate --eval --json --argstr v1 @(v1) --argstr v2 @(v2) --expr '{v1, v2}: builtins.compareVersions v1 v2')) + v1 = Version(v1) + v2 = Version(v2) + if v1 < v2: + return -1 + elif v1 > v2: + return 1 + elif v1 == v2: + return 0 + else: + raise ValueError('these versions are beyond each others celestial plane') def latest_tag_on_branch(branch: str) -> str: @@ -13,16 +26,18 @@ def latest_tag_on_branch(branch: str) -> str: def is_maintenance_branch(branch: str) -> bool: - try: - main_tag = latest_tag_on_branch('main') - current_tag = latest_tag_on_branch(branch) + """ + Returns whether the given branch is probably a maintenance branch. - return version_compare(current_tag, main_tag) < 0 - except subprocess.CalledProcessError: - # This is the case before Lix releases 2.90, since main *has* no - # release tag on it. - # FIXME: delete this case after 2.91 - return False + This uses a heuristic: `main` should have a newer tag than a given + maintenance branch if there has been a major release since that maintenance + branch. + """ + assert remote_is_plausible($(git remote get-url origin).strip()) + main_tag = latest_tag_on_branch('origin/main') + current_tag = latest_tag_on_branch(branch) + + return version_compare(current_tag, main_tag) < 0 def verify_are_on_tag(): From 83247b1c38899a48a7ef683a1d1c14b21cb917d2 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Wed, 7 Aug 2024 20:31:02 -0700 Subject: [PATCH 2/3] releng: clarify/update docs, add instructions after tag This is not a proper fix for the confusion that can happen about how the tags are supposed to be used. For a proper fix, we need to do https://git.lix.systems/lix-project/lix/issues/439 and implement worktrees such that the user never sees the git state anymore. Change-Id: I7b543967f522cede486e42684b48cad47da95429 --- releng/README.md | 36 +++++++++++++++++++++++------------- releng/cli.py | 10 ++++++++-- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/releng/README.md b/releng/README.md index 2aa3b959f..75bd4ab35 100644 --- a/releng/README.md +++ b/releng/README.md @@ -57,12 +57,10 @@ Next, we do the publication with `python -m releng upload`: `nix upgrade-nix`. * s3://releases/lix/lix-VERSION/ gets the following contents * Binary tarballs - * Docs: `manual/` (FIXME: should we actually do this? what about putting it - on docs.lix.systems? I think doing both is correct, since the Web site - should not be an archive of random old manuals) + * Docs: `manual/`, primarily as an archive of old manuals * Docs as tarball in addition to web. * Source tarball - * Docker image (FIXME: upload to forgejo registry and github registry [in the future][upload-docker]) + * Docker image * s3://docs/manual/lix/MAJOR * s3://docs/manual/lix/stable @@ -80,6 +78,7 @@ Next, we do the publication with `python -m releng upload`: FIXME: automate branch-off to `release-*` branch. * **Manually** (FIXME?) switch back to the release branch, which now has the correct revision. +* Deal with the external systems (see sections below). * Post!! * Merge release blog post to [lix-website]. * Toot about it! https://chaos.social/@lix_project @@ -87,22 +86,33 @@ Next, we do the publication with `python -m releng upload`: [lix-website]: https://git.lix.systems/lix-project/lix-website -[upload-docker]: https://git.lix.systems/lix-project/lix/issues/252 - ### Installer -The installer is cross-built to several systems from a Mac using -`build-all.xsh` and `upload-to-lix.xsh` in the installer repo (FIXME: currently -at least; maybe this should be moved here?) . +The installer is cross-built to several systems from a Mac using `build-all.xsh` and `upload-to-lix.xsh` in the installer repo (FIXME: currently at least; maybe this should be moved here?). -It installs a binary tarball (FIXME: [it should be taught to substitute from -cache instead][installer-substitute]) -from some URL; this is the `hydraJobs.binaryTarball`. The default URLs differ -by architecture and are [configured here][tarball-urls]. +It installs a binary tarball (FIXME: [it should be taught to substitute from cache instead][installer-substitute]) from some URL; this is the `hydraJobs.binaryTarball`. +The default URLs differ by architecture and are [configured here][tarball-urls]. + +To automatically do the file changes for a new version, run `python3 set_version.py NEW_VERSION`, and submit the result for review. [installer-substitute]: https://git.lix.systems/lix-project/lix-installer/issues/13 [tarball-urls]: https://git.lix.systems/lix-project/lix-installer/src/commit/693592ed10d421a885bec0a9dd45e87ab87eb90a/src/settings.rs#L14-L28 +### Web site + +The website has various release-version dependent pieces. +You can update them with `python3 update_version.py NEW_VERSION`, which will regenerate the affected page sources. + +These need the release to have been done first as they need hashes for tarballs and such. + +### NixOS module + +The NixOS module has underdeveloped releng in it. +Currently you have to do the whole branch-off dance manually to a `release-VERSION` branch and update the tarball URLs to point to the release versions manually. + +FIXME: this should be unified with the `set_version.py` work in `lix-installer` and probably all the releng kept in here, or kept elsewhere. +Related: https://git.lix.systems/lix-project/lix/issues/439 + ## Infrastructure summary * releases.lix.systems (`s3://releases`): diff --git a/releng/cli.py b/releng/cli.py index f78d4b12d..f2d193444 100644 --- a/releng/cli.py +++ b/releng/cli.py @@ -1,10 +1,13 @@ +import logging +import argparse +import sys + from . import create_release from . import docker from .environment import RelengEnvironment from . import environment -import argparse -import sys +log = logging.getLogger(__name__) def do_build(args): if args.target == 'all': @@ -21,6 +24,9 @@ def do_tag(args): create_release.do_tag_merge(force_tag=args.force_tag, no_check_git=args.no_check_git) + log.info('Merged the release commit into your last branch, and switched to a detached HEAD of the artifact to be released.') + log.info('After you are done with releasing, switch to your previous branch and push that branch for review.') + def do_upload(env: RelengEnvironment, args): create_release.setup_creds(env) From 7246c2d104f12877de7d5b20033346eff88048e6 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Wed, 7 Aug 2024 20:51:25 -0700 Subject: [PATCH 3/3] releng: fix the git push This was broken because gerrit requires that the revision actually is known before it is pushed as a tag. Also, arguably this fixes the original problem mentioned in https://git.lix.systems/lix-project/lix/issues/439 Change-Id: I0373ac01584440f18d32b8da5699bb359cc2c89a --- releng/create_release.xsh | 26 ++++++++++++++++++++++++-- releng/environment.py | 3 +++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/releng/create_release.xsh b/releng/create_release.xsh index 62114350b..d8d6e8a83 100644 --- a/releng/create_release.xsh +++ b/releng/create_release.xsh @@ -2,6 +2,7 @@ import json import subprocess import itertools import textwrap +import logging from pathlib import Path import tempfile import hashlib @@ -15,6 +16,8 @@ from .version import VERSION, RELEASE_NAME, MAJOR, OFFICIAL_RELEASE from .gitutils import verify_are_on_tag, git_preconditions from . import release_notes +log = logging.getLogger(__name__) + $RAISE_SUBPROC_ERROR = True $XONSH_SHOW_TRACEBACK = True @@ -55,6 +58,9 @@ def official_release_commit_tag(force_tag=False): git commit -m @(message) git tag @(['-f'] if force_tag else []) -a -m @(message) @(VERSION) + with open('releng/prev-git-branch.txt', 'w') as fh: + fh.write(prev_branch) + return prev_branch @@ -235,8 +241,24 @@ def upload_artifacts(env: RelengEnvironment, noconfirm=False, no_check_git=False print('[+] Upload manual') upload_manual(env) - print('[+] git push tag') - git push @(['-f'] if force_push_tag else []) @(env.git_repo) f'{VERSION}:refs/tags/{VERSION}' + prev_branch = None + try: + with open('releng/prev-git-branch.txt', 'r') as fh: + prev_branch = fh.read().strip() + except FileNotFoundError: + log.warn('Cannot find previous git branch file, skipping pushing git objects') + + if prev_branch: + print('[+] git push to the repo') + # We have to push the ref to gerrit for review at least such that the + # commit is known, before we can push it as a tag. + if env.git_repo_is_gerrit: + git push @(env.git_repo) f'{prev_branch}:refs/for/{prev_branch}' + else: + git push @(env.git_repo) f'{prev_branch}:{prev_branch}' + + print('[+] git push tag') + git push @(['-f'] if force_push_tag else []) @(env.git_repo) f'{VERSION}:refs/tags/{VERSION}' def do_tag_merge(force_tag=False, no_check_git=False): diff --git a/releng/environment.py b/releng/environment.py index ca8194fe5..2b6554ede 100644 --- a/releng/environment.py +++ b/releng/environment.py @@ -52,6 +52,7 @@ class RelengEnvironment: releases_bucket: str docs_bucket: str git_repo: str + git_repo_is_gerrit: bool docker_targets: list[DockerTarget] @@ -79,6 +80,7 @@ STAGING = RelengEnvironment( cache_store_overlay={'secret-key': 'staging.key'}, releases_bucket='s3://staging-releases', git_repo='ssh://git@git.lix.systems/lix-project/lix-releng-staging', + git_repo_is_gerrit=False, docker_targets=[ # latest will be auto tagged if appropriate DockerTarget('git.lix.systems/lix-project/lix-releng-staging', @@ -113,6 +115,7 @@ PROD = RelengEnvironment( cache_store_overlay={'secret-key': 'prod.key'}, releases_bucket='s3://releases', git_repo=guess_gerrit_remote(), + git_repo_is_gerrit=True, docker_targets=[ # latest will be auto tagged if appropriate DockerTarget('git.lix.systems/lix-project/lix',