lix/releng/create_release.xsh

import json
import subprocess
import itertools
import textwrap
import logging
from pathlib import Path
import tempfile
import hashlib
import datetime

from . import environment
from .environment import RelengEnvironment
from . import keys
from . import docker
from .version import VERSION, RELEASE_NAME, MAJOR, OFFICIAL_RELEASE
from .gitutils import verify_are_on_tag, git_preconditions
from . import release_notes

log = logging.getLogger(__name__)

$RAISE_SUBPROC_ERROR = True
$XONSH_SHOW_TRACEBACK = True

GCROOTS_DIR = Path('./release/gcroots')
BUILT_GCROOTS_DIR = Path('./release/gcroots-build')
DRVS_TXT = Path('./release/drvs.txt')
ARTIFACTS = Path('./release/artifacts')
MANUAL = Path('./release/manual')

RELENG_MSG = "Release created with releng/create_release.xsh"

BUILD_CORES = 16
MAX_JOBS = 2


def setup_creds(env: RelengEnvironment):
    key = keys.get_ephemeral_key(env)
    $AWS_SECRET_ACCESS_KEY = key.secret_key
    $AWS_ACCESS_KEY_ID = key.id
    $AWS_DEFAULT_REGION = 'garage'
    $AWS_ENDPOINT_URL = environment.S3_ENDPOINT


def official_release_commit_tag(force_tag=False):
    print('[+] Setting officialRelease in version.json and tagging')
    prev_branch = $(git symbolic-ref --short HEAD).strip()

    git switch --detach

    # Must be done in two parts due to buffering (opening the file immediately
    # would truncate it).
    new_version_json = $(jq --indent 4 '.official_release = true' version.json)
    with open('version.json', 'w') as fh:
        fh.write(new_version_json)
    git add version.json

    message = f'release: {VERSION} "{RELEASE_NAME}"\n\nRelease produced with releng/create_release.xsh'
    git commit -m @(message)
    git tag @(['-f'] if force_tag else []) -a -m @(message) @(VERSION)

    with open('releng/prev-git-branch.txt', 'w') as fh:
        fh.write(prev_branch)

    return prev_branch


def merge_to_release(prev_branch):
    git switch @(prev_branch)
    # Create a merge back into the release branch so that git tools understand
    # that the release branch contains the tag, without the release commit
    # actually influencing the tree.
    merge_msg = textwrap.dedent("""\
        release: merge release {VERSION} back to mainline

        This merge commit returns to the previous state prior to the release but leaves the tag in the branch history.
        {RELENG_MSG}
    """).format(VERSION=VERSION, RELENG_MSG=RELENG_MSG)
    git merge -m @(merge_msg) -s ours @(VERSION)


def realise(paths: list[str]):
    args = [
        '--realise',
        '--max-jobs',
        MAX_JOBS,
        '--cores',
        BUILD_CORES,
        '--log-format',
        'bar-with-logs',
        '--add-root',
        BUILT_GCROOTS_DIR
    ]
    nix-store @(args) @(paths)


def eval_jobs(build_profile):
    nej_output = $(nix-eval-jobs --workers 4 --gc-roots-dir @(GCROOTS_DIR) --force-recurse --flake f'.#release-jobs.{build_profile}')
    return [json.loads(s) for s in nej_output.strip().split('\n')]


def upload_drv_paths_and_outputs(env: RelengEnvironment, paths: list[str]):
    proc = subprocess.Popen([
            'nix',
            'copy',
            '-v',
            '--to',
            env.cache_store_uri(),
            '--stdin',
        ],
        stdin=subprocess.PIPE,
        env=__xonsh__.env.detype(),
    )

    proc.stdin.write('\n'.join(itertools.chain(paths, x + '^*' for x in paths)).encode())
    proc.stdin.close()
    rv = proc.wait()
    if rv != 0:
        raise subprocess.CalledProcessError(rv, proc.args)


def make_manifest(builds_by_system):

    def manifest_line(system, out):
        return f'  {system} = "{out}";'

    manifest_text = textwrap.dedent("""\
        # This file was generated by releng/create_release.xsh in Lix
        {{
        {lines}
        }}
    """).format(lines='\n'.join(manifest_line(s, p) for (s, p) in builds_by_system.items()))

    return manifest_text


def make_git_tarball(to: Path):
    git archive --verbose --prefix=lix-@(VERSION)/ --format=tar.gz -o @(to) @(VERSION)


def confirm(prompt, expected):
    resp = input(prompt)

    if resp != expected:
        raise ValueError('Unconfirmed')


def sha256_file(f: Path):
    hasher = hashlib.sha256()

    with open(f, 'rb') as h:
        while data := h.read(1024 * 1024):
            hasher.update(data)

    return hasher.hexdigest()


def extract_builds_by_system(eval_result):
    # This could be a dictionary comprehension, but we want to be absolutely
    # sure we don't have duplicates.
    ret = {}
    for attr in eval_result:
        if attr['attrPath'][0] != 'build':
            continue
        assert attr['system'] not in ret
        ret[attr['system']] = attr['outputs']['out']
    return ret


def make_artifacts_dir(eval_result, d: Path):
    d.mkdir(exist_ok=True, parents=True)
    version_dir = d / 'lix' / f'lix-{VERSION}'
    version_dir.mkdir(exist_ok=True, parents=True)

    tarballs_drv = next(p for p in eval_result if p['attr'] == 'tarballs')
    cp --no-preserve=mode -r @(tarballs_drv['outputs']['out'])/* @(version_dir)

    builds_by_system = extract_builds_by_system(eval_result)

    # FIXME: upgrade-nix searches for manifest.nix at root, which is rather annoying
    with open(d / 'manifest.nix', 'w') as h:
        h.write(make_manifest(builds_by_system))

    with open(version_dir / 'manifest.nix', 'w') as h:
        h.write(make_manifest(builds_by_system))

    print('[+] Make sources tarball')

    filename = f'lix-{VERSION}.tar.gz'
    git_tarball = version_dir / filename
    make_git_tarball(git_tarball)

    file_hash = sha256_file(git_tarball)

    print(f'Hash: {file_hash}')
    with open(version_dir / f'{filename}.sha256', 'w') as h:
        h.write(file_hash)


def prepare_release_notes():
    rl_path = release_notes.build_release_notes_to_file()

    commit_msg = textwrap.dedent("""\
        release: release notes for {VERSION}

        {RELENG_MSG}
    """).format(VERSION=VERSION, RELENG_MSG=RELENG_MSG)

    git add @(rl_path) @(release_notes.SUMMARY)
    git rm --ignore-unmatch 'doc/manual/rl-next/*.md'

    git commit -m @(commit_msg)


def upload_artifacts(env: RelengEnvironment, noconfirm=False, no_check_git=False, force_push_tag=False):
    if not no_check_git:
        verify_are_on_tag()
        git_preconditions()
    assert 'AWS_SECRET_ACCESS_KEY' in __xonsh__.env

    tree @(ARTIFACTS)

    env_part = f'environment {env.name}'
    not noconfirm and confirm(
        f'Would you like to release {ARTIFACTS} as {VERSION} in {env.colour(env_part)}? Type "I want to release this to {env.name}" to confirm\n',
        f'I want to release this to {env.name}'
    )

    docker_images = list((ARTIFACTS / f'lix/lix-{VERSION}').glob(f'lix-{VERSION}-docker-image-*.tar.gz'))
    assert docker_images

    print('[+] Upload to cache')
    with open(DRVS_TXT) as fh:
        upload_drv_paths_and_outputs(env, [x.strip() for x in fh.readlines() if x])

    print('[+] Upload docker images')
    for target in env.docker_targets:
        docker.upload_docker_images(target, docker_images)

    print('[+] Upload to release bucket')
    aws s3 cp --recursive @(ARTIFACTS)/ @(env.releases_bucket)/
    print('[+] Upload manual')
    upload_manual(env)

    prev_branch = None
    try:
        with open('releng/prev-git-branch.txt', 'r') as fh:
            prev_branch = fh.read().strip()
    except FileNotFoundError:
        log.warn('Cannot find previous git branch file, skipping pushing git objects')

    if prev_branch:
        print('[+] git push to the repo')
        # We have to push the ref to gerrit for review at least such that the
        # commit is known, before we can push it as a tag.
        if env.git_repo_is_gerrit:
            git push @(env.git_repo) f'{prev_branch}:refs/for/{prev_branch}'
        else:
            git push @(env.git_repo) f'{prev_branch}:{prev_branch}'

        print('[+] git push tag')
        git push @(['-f'] if force_push_tag else []) @(env.git_repo) f'{VERSION}:refs/tags/{VERSION}'


def do_tag_merge(force_tag=False, no_check_git=False):
    if not no_check_git:
        git_preconditions()
    prev_branch = official_release_commit_tag(force_tag=force_tag)
    merge_to_release(prev_branch)
    git switch --detach @(VERSION)


def build_manual(eval_result):
    (drv, manual) = next((x['drvPath'], x['outputs']['doc']) for x in eval_result if x['attr'] == 'build.x86_64-linux')
    print('[+] Building manual')
    realise([drv])

    cp --no-preserve=mode -T -vr @(manual)/share/doc/nix/manual @(MANUAL)


def upload_manual(env: RelengEnvironment):
    if OFFICIAL_RELEASE:
        version = MAJOR
    else:
        version = 'nightly'

    print('[+] aws s3 sync manual')
    aws s3 sync @(MANUAL)/ @(env.docs_bucket)/manual/lix/@(version)/
    if OFFICIAL_RELEASE:
        aws s3 sync @(MANUAL)/ @(env.docs_bucket)/manual/lix/stable/


def build_artifacts(build_profile, no_check_git=False):
    rm -rf release/
    if not no_check_git:
        verify_are_on_tag()
        git_preconditions()

    print('[+] Evaluating')
    eval_result = eval_jobs(build_profile)
    drv_paths = [x['drvPath'] for x in eval_result]

    print('[+] Building')
    realise(drv_paths)
    build_manual(eval_result)

    with open(DRVS_TXT, 'w') as fh:
        # don't bother putting the release tarballs themselves because they are duplicate and huge
        fh.write('\n'.join(x['drvPath'] for x in eval_result if x['attr'] != 'lix-release-tarballs'))

    make_artifacts_dir(eval_result, ARTIFACTS)
    print(f'[+] Done! See {ARTIFACTS}')