Implement docker upload in the releng tools

This uses skopeo to not think about docker daemons. I, however, noticed
that the docker image we had would have totally terrible cache hits, so
I rewrote it.

Fixes: https://git.lix.systems/lix-project/lix/issues/252

Change-Id: I3c5b6c1f3ba0b9dfcac212b2148f390e0cd542b7

package.nix

@@ -39,6 +39,7 @@
   pkg-config,
   python3,
   rapidcheck,
+  skopeo,
   sqlite,
   toml11,
   util-linuxMinimal ? utillinuxMinimal,
@@ -447,6 +448,8 @@ stdenv.mkDerivation (finalAttrs: {
             lib.optional (stdenv.cc.isClang && hostPlatform == buildPlatform) clang-tools_llvm
             ++ [
               pythonEnv
+              # docker image tool
+              skopeo
               just
               nixfmt
               # Load-bearing order. Must come before clang-unwrapped below, but after clang_tools above.

releng/__init__.py

@@ -2,11 +2,12 @@ from xonsh.main import setup
 setup()
 del setup
 
-from releng import environment
-from releng import create_release
-from releng import keys
-from releng import version
-from releng import cli
+from . import environment
+from . import create_release
+from . import keys
+from . import version
+from . import cli
+from . import docker
 
 def reload():
     import importlib
@@ -15,3 +16,4 @@ def reload():
     importlib.reload(keys)
     importlib.reload(version)
     importlib.reload(cli)
+    importlib.reload(docker)

releng/cli.py

@@ -1,4 +1,8 @@
 from . import create_release
+from . import docker
+from .environment import RelengEnvironment
+from . import environment
+import functools
 import argparse
 import sys
 
@@ -18,13 +22,16 @@ def do_tag(args):
                                 no_check_git=args.no_check_git)
 
 
-def do_upload(args):
-    create_release.setup_creds()
+def do_upload(env: RelengEnvironment, args):
+    create_release.setup_creds(env)
     if args.target == 'all':
-        create_release.upload_artifacts(force_push_tag=args.force_push_tag,
-                                        noconfirm=args.noconfirm)
+        docker.check_all_logins(env)
+        create_release.upload_artifacts(env,
+                                        force_push_tag=args.force_push_tag,
+                                        noconfirm=args.noconfirm,
+                                        no_check_git=args.no_check_git)
     elif args.target == 'manual':
-        create_release.upload_manual()
+        create_release.upload_manual(env)
     else:
         raise ValueError('invalid target, unreachable')
 
@@ -77,6 +84,10 @@ def main():
 
     upload = sps.add_parser(
         'upload', help='Upload artifacts to cache and releases bucket')
+    upload.add_argument(
+        '--no-check-git',
+        action='store_true',
+        help="Don't check git state before uploading. For testing.")
     upload.add_argument('--force-push-tag',
                         action='store_true',
                         help='Force push the tag. For testing.')
@@ -90,7 +101,12 @@ def main():
         '--noconfirm',
         action='store_true',
         help="Don't ask for confirmation. For testing/automation.")
-    upload.set_defaults(cmd=do_upload)
+    upload.add_argument('--environment',
+                        choices=list(environment.ENVIRONMENTS.keys()),
+                        default='staging',
+                        help='Environment to release to')
+    upload.set_defaults(cmd=lambda args: do_upload(
+        environment.ENVIRONMENTS[args.environment], args))
 
     args = ap.parse_args()
     args.cmd(args)

releng/create_release.xsh

@@ -7,19 +7,14 @@ import tempfile
 import hashlib
 import datetime
 from . import environment
+from .environment import RelengEnvironment
 from . import keys
+from . import docker
 from .version import VERSION, RELEASE_NAME, MAJOR
 
 $RAISE_SUBPROC_ERROR = True
 $XONSH_SHOW_TRACEBACK = True
 
-RELENG_ENV = environment.STAGING
-
-RELEASES_BUCKET = RELENG_ENV.releases_bucket
-DOCS_BUCKET = RELENG_ENV.docs_bucket
-CACHE_STORE = RELENG_ENV.cache_store_uri()
-REPO = RELENG_ENV.git_repo
-
 GCROOTS_DIR = Path('./release/gcroots')
 BUILT_GCROOTS_DIR = Path('./release/gcroots-build')
 DRVS_TXT = Path('./release/drvs.txt')
@@ -35,8 +30,8 @@ MAX_JOBS = 2
 RELEASE_SYSTEMS = ["x86_64-linux"]
 
 
-def setup_creds():
-    key = keys.get_ephemeral_key(RELENG_ENV)
+def setup_creds(env: RelengEnvironment):
+    key = keys.get_ephemeral_key(env)
     $AWS_SECRET_ACCESS_KEY = key.secret_key
     $AWS_ACCESS_KEY_ID = key.id
     $AWS_DEFAULT_REGION = 'garage'
@@ -102,13 +97,13 @@ def eval_jobs():
     ]
 
 
-def upload_drv_paths_and_outputs(paths: list[str]):
+def upload_drv_paths_and_outputs(env: RelengEnvironment, paths: list[str]):
     proc = subprocess.Popen([
             'nix',
             'copy',
             '-v',
             '--to',
-            CACHE_STORE,
+            env.cache_store_uri(),
             '--stdin',
         ],
         stdin=subprocess.PIPE,
@@ -250,7 +245,10 @@ def verify_are_on_tag():
     assert current_tag == VERSION
 
 
-def upload_artifacts(noconfirm=False, force_push_tag=False):
+def upload_artifacts(env: RelengEnvironment, noconfirm=False, no_check_git=False, force_push_tag=False):
+    if not no_check_git:
+        verify_are_on_tag()
+        git_preconditions()
     assert 'AWS_SECRET_ACCESS_KEY' in __xonsh__.env
 
     tree @(ARTIFACTS)
@@ -262,16 +260,21 @@ def upload_artifacts(noconfirm=False, force_push_tag=False):
 
     print('[+] Upload to cache')
     with open(DRVS_TXT) as fh:
-        upload_drv_paths_and_outputs([x.strip() for x in fh.readlines() if x])
+        upload_drv_paths_and_outputs(env, [x.strip() for x in fh.readlines() if x])
 
+    docker_images = (ARTIFACTS / f'lix/lix-{VERSION}').glob(f'lix-{VERSION}-docker-image-*.tar.gz')
+    print('[+] Upload docker images')
+    for image in docker_images:
+        for target in env.docker_targets:
+            docker.upload_docker_image(target, image)
 
     print('[+] Upload to release bucket')
-    aws s3 cp --recursive @(ARTIFACTS)/ @(RELEASES_BUCKET)/
+    aws s3 cp --recursive @(ARTIFACTS)/ @(env.releases_bucket)/
     print('[+] Upload manual')
-    upload_manual()
+    upload_manual(env)
 
     print('[+] git push tag')
-    git push @(['-f'] if force_push_tag else []) @(REPO) f'{VERSION}:refs/tags/{VERSION}'
+    git push @(['-f'] if force_push_tag else []) @(env.git_repo) f'{VERSION}:refs/tags/{VERSION}'
 
 
 def do_tag_merge(force_tag=False, no_check_git=False):
@@ -290,7 +293,7 @@ def build_manual(eval_result):
     cp --no-preserve=mode -vr @(manual)/share/doc/nix @(MANUAL)
 
 
-def upload_manual():
+def upload_manual(env: RelengEnvironment):
     stable = json.loads($(nix eval --json '.#nix.officialRelease'))
     if stable:
         version = MAJOR
@@ -298,9 +301,9 @@ def upload_manual():
         version = 'nightly'
 
     print('[+] aws s3 sync manual')
-    aws s3 sync @(MANUAL)/ @(DOCS_BUCKET)/manual/lix/@(version)/
+    aws s3 sync @(MANUAL)/ @(env.docs_bucket)/manual/lix/@(version)/
     if stable:
-        aws s3 sync @(MANUAL)/ @(DOCS_BUCKET)/manual/lix/stable/
+        aws s3 sync @(MANUAL)/ @(env.docs_bucket)/manual/lix/stable/
 
 
 def build_artifacts(no_check_git=False):

releng/docker.xsh

@@ -0,0 +1,13 @@
+from .environment import DockerTarget, RelengEnvironment
+from .version import VERSION
+from pathlib import Path
+
+def check_all_logins(env: RelengEnvironment):
+    for target in env.docker_targets:
+        check_login(target)
+
+def check_login(target: DockerTarget):
+    skopeo login @(target.registry_name())
+
+def upload_docker_image(target: DockerTarget, path: Path):
+    skopeo --insecure-policy copy docker-archive:@(path) docker://@(target.resolve(version=VERSION))

releng/environment.py

@@ -16,6 +16,21 @@ DEFAULT_STORE_URI_BITS = {
 }
 
 
+@dataclasses.dataclass
+class DockerTarget:
+    registry_path: str
+
+    def resolve(self, version: str) -> str:
+        """Applies templates:
+        - version: the Lix version
+        """
+        return self.registry_path.format(version=version)
+
+    def registry_name(self) -> str:
+        [a, _, _] = self.registry_path.partition('/')
+        return a
+
+
 @dataclasses.dataclass
 class RelengEnvironment:
     name: str
@@ -26,22 +41,33 @@ class RelengEnvironment:
     docs_bucket: str
     git_repo: str
 
+    docker_targets: list[DockerTarget]
+
     def cache_store_uri(self):
         qs = DEFAULT_STORE_URI_BITS.copy()
         qs.update(self.cache_store_overlay)
         return self.cache_bucket + "?" + urllib.parse.urlencode(qs)
 
+
 STAGING = RelengEnvironment(
     name='staging',
     docs_bucket='s3://staging-docs',
     cache_bucket='s3://staging-cache',
-    cache_store_overlay={
-        'secret-key': 'staging.key'
-    },
+    cache_store_overlay={'secret-key': 'staging.key'},
     releases_bucket='s3://staging-releases',
     git_repo='ssh://git@git.lix.systems/lix-project/lix-releng-staging',
+    docker_targets=[
+        DockerTarget(
+            'git.lix.systems/lix-project/lix-releng-staging:{version}'),
+        DockerTarget(
+            'ghcr.io/lix-project/lix-releng-staging:{version}'),
+    ],
 )
 
+ENVIRONMENTS = {
+    'staging': STAGING,
+}
+
 
 @dataclasses.dataclass
 class S3Credentials: