From e007eb480c6041fd98b8f9e53bdac2ba82e4648c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 1 Jul 2022 15:02:00 +0200 Subject: [PATCH] dockerTools.buildImage: Add copyToRoot to replace contents, explain usage --- doc/builders/images/dockertools.section.md | 15 ++- .../from_md/release-notes/rl-2211.section.xml | 9 ++ .../manual/release-notes/rl-2211.section.md | 3 + nixos/tests/docker-tools-cross.nix | 6 +- pkgs/build-support/docker/default.nix | 38 +++++--- pkgs/build-support/docker/examples.nix | 97 ++++++++++++++----- 6 files changed, 128 insertions(+), 40 deletions(-) diff --git a/doc/builders/images/dockertools.section.md b/doc/builders/images/dockertools.section.md index d7f8741437cc..2a41d48cf134 100644 --- a/doc/builders/images/dockertools.section.md +++ b/doc/builders/images/dockertools.section.md @@ -20,7 +20,12 @@ buildImage { fromImageName = null; fromImageTag = "latest"; - contents = pkgs.redis; + copyToRoot = pkgs.buildEnv { + name = "image-root"; + paths = [ pkgs.redis ]; + pathsToLink = [ "/bin" ]; + }; + runAsRoot = '' #!${pkgs.runtimeShell} mkdir -p /data @@ -46,7 +51,7 @@ The above example will build a Docker image `redis/latest` from the given base i - `fromImageTag` can be used to further specify the tag of the base image within the repository, in case an image contains multiple tags. By default it's `null`, in which case `buildImage` will peek the first tag available for the base image. -- `contents` is a derivation that will be copied in the new layer of the resulting image. This can be similarly seen as `ADD contents/ /` in a `Dockerfile`. By default it's `null`. +- `copyToRoot` is a derivation that will be copied in the new layer of the resulting image. This can be similarly seen as `ADD contents/ /` in a `Dockerfile`. By default it's `null`. - `runAsRoot` is a bash script that will run as root in an environment that overlays the existing layers of the base image with the new resulting layer, including the previously copied `contents` derivation. This can be similarly seen as `RUN ...` in a `Dockerfile`. @@ -81,7 +86,11 @@ pkgs.dockerTools.buildImage { name = "hello"; tag = "latest"; created = "now"; - contents = pkgs.hello; + copyToRoot = pkgs.buildEnv { + name = "image-root"; + paths = [ pkgs.hello ]; + pathsToLink = [ "/bin" ]; + }; config.Cmd = [ "/bin/hello" ]; } diff --git a/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml index a11c2bb61ffb..91d169d42ced 100644 --- a/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml +++ b/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml @@ -296,6 +296,15 @@ and require manual remediation. + + + dockerTools.buildImage deprecates the + misunderstood contents parameter, in favor + of copyToRoot. Use + copyToRoot = buildEnv { ... }; or similar + if you intend to add packages to /bin. + + memtest86+ was updated from 5.00-coreboot-002 to 6.00-beta2. diff --git a/nixos/doc/manual/release-notes/rl-2211.section.md b/nixos/doc/manual/release-notes/rl-2211.section.md index 275c522a54f1..3f799a2ad68f 100644 --- a/nixos/doc/manual/release-notes/rl-2211.section.md +++ b/nixos/doc/manual/release-notes/rl-2211.section.md @@ -112,6 +112,9 @@ Use `configure.packages` instead. - Matrix Synapse now requires entries in the `state_group_edges` table to be unique, in order to prevent accidentally introducing duplicate information (for example, because a database backup was restored multiple times). If your Synapse database already has duplicate rows in this table, this could fail with an error and require manual remediation. +- `dockerTools.buildImage` deprecates the misunderstood `contents` parameter, in favor of `copyToRoot`. + Use `copyToRoot = buildEnv { ... };` or similar if you intend to add packages to `/bin`. + - memtest86+ was updated from 5.00-coreboot-002 to 6.00-beta2. It is now the upstream version from https://www.memtest.org/, as coreboot's fork is no longer available. - There is a new module for the `thunar` program (the Xfce file manager), which depends on the `xfconf` dbus service, and also has a dbus service and a systemd unit. The option `services.xserver.desktopManager.xfce.thunarPlugins` has been renamed to `programs.thunar.plugins`, and in a future release it may be removed. diff --git a/nixos/tests/docker-tools-cross.nix b/nixos/tests/docker-tools-cross.nix index 8791ec258127..14cb14ceeaea 100644 --- a/nixos/tests/docker-tools-cross.nix +++ b/nixos/tests/docker-tools-cross.nix @@ -24,7 +24,11 @@ let hello1 = remoteCrossPkgs.dockerTools.buildImage { name = "hello1"; tag = "latest"; - contents = remoteCrossPkgs.hello; + copyToRoot = remoteCrossPkgs.buildEnv { + name = "image-root"; + pathsToLink = [ "/bin" ]; + paths = [ remoteCrossPkgs.hello ]; + }; }; hello2 = remoteCrossPkgs.dockerTools.buildLayeredImage { diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix index 75e0d52d921b..39008df74f17 100644 --- a/pkgs/build-support/docker/default.nix +++ b/pkgs/build-support/docker/default.nix @@ -332,7 +332,7 @@ rec { , # JSON containing configuration and metadata for this layer. baseJson , # Files to add to the layer. - contents ? null + copyToRoot ? null , # When copying the contents into the image, preserve symlinks to # directories (see `rsync -K`). Otherwise, transform those symlinks # into directories. @@ -344,7 +344,8 @@ rec { }: runCommand "docker-layer-${name}" { - inherit baseJson contents extraCommands; + inherit baseJson extraCommands; + contents = copyToRoot; nativeBuildInputs = [ jshon rsync tarsum ]; } '' @@ -390,7 +391,8 @@ rec { , # Script to run as root. Bash. runAsRoot , # Files to add to the layer. If null, an empty layer will be created. - contents ? null + # To add packages to /bin, use `buildEnv` or similar. + copyToRoot ? null , # When copying the contents into the image, preserve symlinks to # directories (see `rsync -K`). Otherwise, transform those symlinks # into directories. @@ -418,9 +420,9 @@ rec { inherit fromImage fromImageName fromImageTag diskSize; - preMount = lib.optionalString (contents != null && contents != [ ]) '' + preMount = lib.optionalString (copyToRoot != null && copyToRoot != [ ]) '' echo "Adding contents..." - for item in ${escapeShellArgs (map (c: "${c}") (toList contents))}; do + for item in ${escapeShellArgs (map (c: "${c}") (toList copyToRoot))}; do echo "Adding $item..." rsync -a${if keepContentsDirlinks then "K" else "k"} --chown=0:0 $item/ layer/ done @@ -500,7 +502,7 @@ rec { , # Tag of the parent image; will be read from the image otherwise. fromImageTag ? null , # Files to put on the image (a nix store path or list of paths). - contents ? null + copyToRoot ? null , # When copying the contents into the image, preserve symlinks to # directories (see `rsync -K`). Otherwise, transform those symlinks # into directories. @@ -517,10 +519,20 @@ rec { diskSize ? 1024 , # Time of creation of the image. created ? "1970-01-01T00:00:01Z" + , # Deprecated. + contents ? null , }: let + checked = + lib.warnIf (contents != null) + "in docker image ${name}: The contents parameter is deprecated. Change to copyToRoot if the contents are designed to be copied to the root filesystem, such as when you use `buildEnv` or similar between contents and your packages. Use copyToRoot = buildEnv { ... }; or similar if you intend to add packages to /bin." + lib.throwIf (contents != null && copyToRoot != null) "in docker image ${name}: You can not specify both contents and copyToRoot." + ; + + rootContents = if copyToRoot == null then contents else copyToRoot; + baseName = baseNameOf name; # Create a JSON blob of the configuration. Set the date to unix zero. @@ -545,13 +557,15 @@ rec { mkPureLayer { name = baseName; - inherit baseJson contents keepContentsDirlinks extraCommands uid gid; + inherit baseJson keepContentsDirlinks extraCommands uid gid; + copyToRoot = rootContents; } else mkRootLayer { name = baseName; inherit baseJson fromImage fromImageName fromImageTag - contents keepContentsDirlinks runAsRoot diskSize + keepContentsDirlinks runAsRoot diskSize extraCommands; + copyToRoot = rootContents; }; result = runCommand "docker-image-${baseName}.tar.gz" { @@ -715,7 +729,7 @@ rec { ''; in - result; + checked result; # Merge the tarballs of images built with buildImage into a single # tarball that contains all images. Running `docker load` on the resulting @@ -776,12 +790,14 @@ rec { # contents. The main purpose is to be able to use nix commands in # the container. # Be careful since this doesn't work well with multilayer. - buildImageWithNixDb = args@{ contents ? null, extraCommands ? "", ... }: ( + # TODO: add the dependencies of the config json. + buildImageWithNixDb = args@{ copyToRoot ? contents, contents ? null, extraCommands ? "", ... }: ( buildImage (args // { - extraCommands = (mkDbExtraCommand contents) + extraCommands; + extraCommands = (mkDbExtraCommand copyToRoot) + extraCommands; }) ); + # TODO: add the dependencies of the config json. buildLayeredImageWithNixDb = args@{ contents ? null, extraCommands ? "", ... }: ( buildLayeredImage (args // { extraCommands = (mkDbExtraCommand contents) + extraCommands; diff --git a/pkgs/build-support/docker/examples.nix b/pkgs/build-support/docker/examples.nix index f0535f59dfcc..f92a55c2e4f7 100644 --- a/pkgs/build-support/docker/examples.nix +++ b/pkgs/build-support/docker/examples.nix @@ -24,7 +24,11 @@ rec { bash = buildImage { name = "bash"; tag = "latest"; - contents = pkgs.bashInteractive; + copyToRoot = pkgs.buildEnv { + name = "image-root"; + paths = [ pkgs.bashInteractive ]; + pathsToLink = [ "/bin" ]; + }; }; # 2. service example, layered on another image @@ -36,7 +40,12 @@ rec { fromImage = bash; # fromImage = debian; - contents = pkgs.redis; + copyToRoot = pkgs.buildEnv { + name = "image-root"; + paths = [ pkgs.redis ]; + pathsToLink = [ "/bin" ]; + }; + runAsRoot = '' mkdir -p /data ''; @@ -118,13 +127,17 @@ rec { # 5. example of multiple contents, emacs and vi happily coexisting editors = buildImage { name = "editors"; - contents = [ - pkgs.coreutils - pkgs.bash - pkgs.emacs - pkgs.vim - pkgs.nano - ]; + copyToRoot = pkgs.buildEnv { + name = "image-root"; + pathsToLink = [ "/bin" ]; + paths = [ + pkgs.coreutils + pkgs.bash + pkgs.emacs + pkgs.vim + pkgs.nano + ]; + }; }; # 6. nix example to play with the container nix store @@ -132,13 +145,17 @@ rec { nix = buildImageWithNixDb { name = "nix"; tag = "latest"; - contents = [ - # nix-store uses cat program to display results as specified by - # the image env variable NIX_PAGER. - pkgs.coreutils - pkgs.nix - pkgs.bash - ]; + copyToRoot = pkgs.buildEnv { + name = "image-root"; + pathsToLink = [ "/bin" ]; + paths = [ + # nix-store uses cat program to display results as specified by + # the image env variable NIX_PAGER. + pkgs.coreutils + pkgs.nix + pkgs.bash + ]; + }; config = { Env = [ "NIX_PAGER=cat" @@ -155,7 +172,11 @@ rec { name = "onTopOfPulledImage"; tag = "latest"; fromImage = nixFromDockerHub; - contents = [ pkgs.hello ]; + copyToRoot = pkgs.buildEnv { + name = "image-root"; + pathsToLink = [ "/bin" ]; + paths = [ pkgs.hello ]; + }; }; # 8. regression test for erroneous use of eval and string expansion. @@ -163,7 +184,11 @@ rec { runAsRootExtraCommands = pkgs.dockerTools.buildImage { name = "runAsRootExtraCommands"; tag = "latest"; - contents = [ pkgs.coreutils ]; + copyToRoot = pkgs.buildEnv { + name = "image-root"; + pathsToLink = [ "/bin" ]; + paths = [ pkgs.coreutils ]; + }; # The parens here are to create problematic bash to embed and eval. In case # this is *embedded* into the script (with nix expansion) the initial quotes # will close the string and the following parens are unexpected @@ -176,7 +201,11 @@ rec { unstableDate = pkgs.dockerTools.buildImage { name = "unstable-date"; tag = "latest"; - contents = [ pkgs.coreutils ]; + copyToRoot = pkgs.buildEnv { + name = "image-root"; + pathsToLink = [ "/bin" ]; + paths = [ pkgs.coreutils ]; + }; created = "now"; }; @@ -265,7 +294,11 @@ rec { name = "l3"; fromImage = l2; tag = "latest"; - contents = [ pkgs.coreutils ]; + copyToRoot = pkgs.buildEnv { + name = "image-root"; + pathsToLink = [ "/bin" ]; + paths = [ pkgs.coreutils ]; + }; extraCommands = '' mkdir -p tmp echo layer3 > tmp/layer3 @@ -290,7 +323,11 @@ rec { name = "child"; fromImage = environmentVariablesParent; tag = "latest"; - contents = [ pkgs.coreutils ]; + copyToRoot = pkgs.buildEnv { + name = "image-root"; + pathsToLink = [ "/bin" ]; + paths = [ pkgs.coreutils ]; + }; config = { Env = [ "FROM_CHILD=true" @@ -424,7 +461,11 @@ rec { name = "layers-unpack-order-${layerName}"; tag = "latest"; fromImage = parent; - contents = [ pkgs.coreutils ]; + copyToRoot = pkgs.buildEnv { + name = "image-root"; + pathsToLink = [ "/bin" ]; + paths = [ pkgs.coreutils ]; + }; runAsRoot = '' #!${pkgs.runtimeShell} echo -n "${layerName}" >> /layer-order @@ -441,7 +482,8 @@ rec { # buildImage without explicit tag bashNoTag = pkgs.dockerTools.buildImage { name = "bash-no-tag"; - contents = pkgs.bashInteractive; + # Not recommended. Use `buildEnv` between copy and packages to avoid file duplication. + copyToRoot = pkgs.bashInteractive; }; # buildLayeredImage without explicit tag @@ -501,7 +543,11 @@ rec { in crossPkgs.dockerTools.buildImage { name = "hello-cross"; tag = "latest"; - contents = crossPkgs.hello; + copyToRoot = pkgs.buildEnv { + name = "image-root"; + pathsToLink = [ "/bin" ]; + paths = [ crossPkgs.hello ]; + }; }; # layered image where a store path is itself a symlink @@ -643,7 +689,8 @@ rec { build-image-with-path = buildImage { name = "build-image-with-path"; tag = "latest"; - contents = [ pkgs.bashInteractive ./test-dummy ]; + # Not recommended. Use `buildEnv` between copy and packages to avoid file duplication. + copyToRoot = [ pkgs.bashInteractive ./test-dummy ]; }; layered-image-with-path = pkgs.dockerTools.streamLayeredImage {