Add a post-build-hook

Passing `--post-build-hook /foo/bar` to a nix-* command will cause
`/foo/bar` to be executed after each build with the following
environment variables set:

    DRV_PATH=/nix/store/drv-that-has-been-built.drv
    OUT_PATHS=/nix/store/...build /nix/store/...build-bin /nix/store/...build-dev

This can be useful in particular to upload all the builded artifacts to
the cache (including the ones that don't appear in the runtime closure
of the final derivation or are built because of IFD).

This new feature prints the stderr/stdout output to the `nix-build`
and `nix build` client, and the output is printed in a Nix 2
compatible format:

    [nix]$ ./inst/bin/nix-build ./test.nix
    these derivations will be built:
      /nix/store/ishzj9ni17xq4hgrjvlyjkfvm00b0ch9-my-example-derivation.drv
    building '/nix/store/ishzj9ni17xq4hgrjvlyjkfvm00b0ch9-my-example-derivation.drv'...
    hello!
    bye!
    running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'...
    post-build-hook: + sleep 1
    post-build-hook: + echo 'Signing paths' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation
    post-build-hook: Signing paths /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation
    post-build-hook: + sleep 1
    post-build-hook: + echo 'Uploading paths' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation
    post-build-hook: Uploading paths /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation
    post-build-hook: + sleep 1
    post-build-hook: + printf 'very important stuff'
    /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation

    [nix-shell:~/projects/github.com/NixOS/nix]$ ./inst/bin/nix build -L -f ./test.nix
    my-example-derivation> hello!
    my-example-derivation> bye!
    my-example-derivation (post)> + sleep 1
    my-example-derivation (post)> + echo 'Signing paths' /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation
    my-example-derivation (post)> Signing paths /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation
    my-example-derivation (post)> + sleep 1
    my-example-derivation (post)> + echo 'Uploading paths' /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation
    my-example-derivation (post)> Uploading paths /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation
    my-example-derivation (post)> + sleep 1
    my-example-derivation (post)> + printf 'very important stuff'
    [1 built, 0.0 MiB DL]

Co-authored-by: Graham Christensen <graham@grahamc.com>
Co-authored-by: Eelco Dolstra <edolstra@gmail.com>

doc/manual/advanced-topics/advanced-topics.xml

@@ -8,5 +8,6 @@
 
 <xi:include href="distributed-builds.xml" />
 <xi:include href="diff-hook.xml" />
+<xi:include href="post-build-hook.xml" />
 
 </part>

doc/manual/advanced-topics/post-build-hook.xml

@@ -0,0 +1,157 @@
+<chapter xmlns="http://docbook.org/ns/docbook"
+      xmlns:xlink="http://www.w3.org/1999/xlink"
+      xmlns:xi="http://www.w3.org/2001/XInclude"
+      xml:id="chap-post-build-hook"
+      version="5.0"
+      >
+
+<title>Using the <xref linkend="conf-post-build-hook" /></title>
+<subtitle>Uploading to an S3-compatible binary cache after each build</subtitle>
+
+
+<section xml:id="chap-post-build-hook-caveats">
+  <title>Implementation Caveats</title>
+  <para>Here we use the post-build hook to upload to a binary cache.
+  This is a simple and working example, but it is not suitable for all
+  use cases.</para>
+
+  <para>The post build hook program runs after each executed build,
+  and blocks the build loop. The build loop exits if the hook program
+  fails.</para>
+
+  <para>Concretely, this implementation will make Nix slow or unusable
+  when the internet is slow or unreliable.</para>
+
+  <para>A more advanced implementation might pass the store paths to a
+  user-supplied daemon or queue for processing the store paths outside
+  of the build loop.</para>
+</section>
+
+<section>
+  <title>Prerequisites</title>
+
+  <para>
+    This tutorial assumes you have configured an S3-compatible binary cache
+    according to the instructions at
+    <xref linkend="ssec-s3-substituter-authenticated-writes" />, and
+    that the <literal>root</literal> user's default AWS profile can
+    upload to the bucket.
+  </para>
+</section>
+
+<section>
+  <title>Set up a Signing Key</title>
+  <para>Use <command>nix-store --generate-binary-cache-key</command> to
+  create our public and private signing keys. We will sign paths
+  with the private key, and distribute the public key for verifying
+  the authenticity of the paths.</para>
+
+  <screen>
+# nix-store --generate-binary-cache-key example-nix-cache-1 /etc/nix/key.private /etc/nix/key.public
+# cat /etc/nix/key.public
+example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM=
+</screen>
+
+<para>Then, add the public key and the cache URL to your
+<filename>nix.conf</filename>'s <xref linkend="conf-trusted-public-keys" />
+and <xref linkend="conf-substituters" /> like:</para>
+
+<programlisting>
+substituters = https://cache.nixos.org/ s3://example-nix-cache
+trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM=
+</programlisting>
+
+<para>we will restart the Nix daemon a later step.</para>
+</section>
+
+<section>
+  <title>Implementing the build hook</title>
+  <para>Write the following script to
+  <filename>/etc/nix/upload-to-cache.sh</filename>:
+  </para>
+
+  <programlisting>
+#!/bin/sh
+
+set -eu
+
+echo "Signing paths" $OUT_PATHS
+nix sign-paths --key-file /etc/nix/key.private $OUT_PATHS
+echo "Uploading paths" $OUT_PATHS
+exec nix copy --to 's3://example-nix-cache' $OUT_PATHS
+</programlisting>
+
+  <note>
+    <title>Should <literal>$OUT_PATHS</literal> be quoted?</title>
+    <para>
+      The <literal>$OUT_PATHS</literal> variable is a space-separated
+      list of Nix store paths. In this case, we expect and want the
+      shell to perform word splitting to make each output path its
+      own argument to <command>nix sign-paths</command>. Nix guarantees
+      the paths will only contain characters which are safe for word
+      splitting, and free of any globs.
+    </para>
+  </note>
+  <para>
+    Then make sure the hook program is executable by the <literal>root</literal> user:
+    <screen>
+# chmod +x /etc/nix/upload-to-cache.sh
+</screen></para>
+</section>
+
+<section>
+  <title>Updating Nix Configuration</title>
+
+  <para>Edit <filename>/etc/nix/nix.conf</filename> to run our hook,
+  by adding the following configuration snippet at the end:</para>
+
+  <programlisting>
+post-build-hook = /etc/nix/upload-to-cache.sh
+</programlisting>
+
+<para>Then, restart the <command>nix-daemon</command>.</para>
+</section>
+
+<section>
+  <title>Testing</title>
+
+  <para>Build any derivation, for example:</para>
+
+  <screen>
+$ nix-build -E '(import &lt;nixpkgs&gt; {}).writeText "example" (builtins.toString builtins.currentTime)'
+these derivations will be built:
+  /nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv
+building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'...
+running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'...
+post-build-hook: Signing paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+post-build-hook: Uploading paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+/nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+</screen>
+
+  <para>Then delete the path from the store, and try substituting it from the binary cache:</para>
+  <screen>
+$ rm ./result
+$ nix-store --delete /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+</screen>
+
+<para>Now, copy the path back from the cache:</para>
+<screen>
+$ nix store --realize /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+copying path '/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example from 's3://example-nix-cache'...
+warning: you did not specify '--add-root'; the result might be removed by the garbage collector
+/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example
+</screen>
+</section>
+<section>
+  <title>Conclusion</title>
+  <para>
+    We now have a Nix installation configured to automatically sign and
+    upload every local build to a remote binary cache.
+  </para>
+
+  <para>
+    Before deploying this to production, be sure to consider the
+    implementation caveats in <xref linkend="chap-post-build-hook-caveats" />.
+  </para>
+</section>
+</chapter>

doc/manual/command-ref/conf-file.xml

@@ -656,6 +656,61 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
+  <varlistentry xml:id="conf-post-build-hook">
+    <term><literal>post-build-hook</literal></term>
+    <listitem>
+      <para>Optional. The path to a program to execute after each build.</para>
+
+      <para>This option is only settable in the global
+      <filename>nix.conf</filename>, or on the command line by trusted
+      users.</para>
+
+      <para>When using the nix-daemon, the daemon executes the hook as
+      <literal>root</literal>. If the nix-daemon is not involved, the
+      hook runs as the user executing the nix-build.</para>
+
+      <itemizedlist>
+        <listitem><para>The hook executes after an evaluation-time build.</para></listitem>
+        <listitem><para>The hook does not execute on substituted paths.</para></listitem>
+        <listitem><para>The hook's output always goes to the user's terminal.</para></listitem>
+        <listitem><para>If the hook fails, the build succeeds but no further builds execute.</para></listitem>
+      </itemizedlist>
+
+      <para>The program executes with no arguments. The program's environment
+      contains the following environment variables:</para>
+
+      <variablelist>
+        <varlistentry>
+          <term><envar>DRV_PATH</envar></term>
+          <listitem>
+            <para>The derivation for the built paths.</para>
+            <para>Example:
+            <literal>/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv</literal>
+            </para>
+          </listitem>
+        </varlistentry>
+
+        <varlistentry>
+          <term><envar>OUT_PATHS</envar></term>
+          <listitem>
+            <para>Output paths of the built derivation, separated by a space (<literal> </literal>) character.</para>
+            <para>Example:
+            <literal>/nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev
+            /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc
+            /nix/store/6bqvbzjkcp9695dq0dpl5y43nvy37pq1-bash-4.4-p23-info
+            /nix/store/r7fng3kk3vlpdlh2idnrbn37vh4imlj2-bash-4.4-p23-man
+            /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23</literal>.
+            </para>
+          </listitem>
+        </varlistentry>
+      </variablelist>
+
+      <para>See <xref linkend="chap-post-build-hook" /> for an example
+      implementation.</para>
+
+    </listitem>
+  </varlistentry>
+
   <varlistentry xml:id="conf-repeat"><term><literal>repeat</literal></term>
 
     <listitem><para>How many times to repeat builds to check whether

src/libstore/build.cc

@@ -1629,6 +1629,61 @@ void DerivationGoal::buildDone()
            being valid. */
         registerOutputs();
 
+        if (settings.postBuildHook != "") {
+            Activity act(*logger, lvlInfo, actPostBuildHook,
+                fmt("running post-build-hook '%s'", settings.postBuildHook),
+                Logger::Fields{drvPath});
+            PushActivity pact(act.id);
+            auto outputPaths = drv->outputPaths();
+            std::map<std::string, std::string> hookEnvironment = getEnv();
+
+            hookEnvironment.emplace("DRV_PATH", drvPath);
+            hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", outputPaths)));
+
+            RunOptions opts(settings.postBuildHook, {});
+            opts.environment = hookEnvironment;
+
+            struct LogSink : Sink {
+                Activity & act;
+                std::string currentLine;
+
+                LogSink(Activity & act) : act(act) { }
+
+                void operator() (const unsigned char * data, size_t len) override {
+                    for (size_t i = 0; i < len; i++) {
+                        auto c = data[i];
+
+                        if (c == '\n') {
+                            flushLine();
+                        } else {
+                            currentLine += c;
+                        }
+                    }
+                }
+
+                void flushLine() {
+                    if (settings.verboseBuild) {
+                        printError("post-build-hook: " + currentLine);
+                    } else {
+                        act.result(resPostBuildLogLine, currentLine);
+                    }
+                    currentLine.clear();
+                }
+
+                ~LogSink() {
+                    if (currentLine != "") {
+                        currentLine += '\n';
+                        flushLine();
+                    }
+                }
+            };
+            LogSink sink(act);
+
+            opts.standardOut = &sink;
+            opts.mergeStderrToStdout = true;
+            runProgram2(opts);
+        }
+
         if (buildMode == bmCheck) {
             done(BuildResult::Built);
             return;

src/libstore/globals.hh

@@ -315,6 +315,9 @@ public:
         "pre-build-hook",
         "A program to run just before a build to set derivation-specific build settings."};
 
+    Setting<std::string> postBuildHook{this, "", "post-build-hook",
+        "A program to run just after each succesful build."};
+
     Setting<std::string> netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file",
         "Path to the netrc file used to obtain usernames/passwords for downloads."};
 

src/libutil/logging.hh

@@ -26,6 +26,7 @@ typedef enum {
     actVerifyPaths = 107,
     actSubstitute = 108,
     actQueryPathInfo = 109,
+    actPostBuildHook = 110,
 } ActivityType;
 
 typedef enum {
@@ -36,6 +37,7 @@ typedef enum {
     resSetPhase = 104,
     resProgress = 105,
     resSetExpected = 106,
+    resPostBuildLogLine = 107,
 } ResultType;
 
 typedef uint64_t ActivityId;

src/libutil/util.cc

@@ -84,6 +84,15 @@ void clearEnv()
         unsetenv(name.first.c_str());
 }
 
+void replaceEnv(std::map<std::string, std::string> newEnv)
+{
+    clearEnv();
+    for (auto newEnvVar : newEnv)
+    {
+        setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1);
+    }
+}
+
 
 Path absPath(Path path, Path dir)
 {
@@ -1019,10 +1028,22 @@ void runProgram2(const RunOptions & options)
     if (options.standardOut) out.create();
     if (source) in.create();
 
+    ProcessOptions processOptions;
+    // vfork implies that the environment of the main process and the fork will
+    // be shared (technically this is undefined, but in practice that's the
+    // case), so we can't use it if we alter the environment
+    if (options.environment)
+        processOptions.allowVfork = false;
+
     /* Fork. */
     Pid pid = startProcess([&]() {
+        if (options.environment)
+            replaceEnv(*options.environment);
         if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
             throw SysError("dupping stdout");
+        if (options.mergeStderrToStdout)
+            if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1)
+                throw SysError("cannot dup stdout into stderr");
         if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1)
             throw SysError("dupping stdin");
 
@@ -1047,7 +1068,7 @@ void runProgram2(const RunOptions & options)
             execv(options.program.c_str(), stringsToCharPtrs(args_).data());
 
         throw SysError("executing '%1%'", options.program);
-    });
+    }, processOptions);
 
     out.writeSide = -1;
 

src/libutil/util.hh

@@ -270,12 +270,14 @@ struct RunOptions
     std::optional<uid_t> uid;
     std::optional<uid_t> gid;
     std::optional<Path> chdir;
+    std::optional<std::map<std::string, std::string>> environment;
     Path program;
     bool searchPath = true;
     Strings args;
     std::optional<std::string> input;
     Source * standardIn = nullptr;
     Sink * standardOut = nullptr;
+    bool mergeStderrToStdout = false;
     bool _killStderr = false;
 
     RunOptions(const Path & program, const Strings & args)

src/nix/progress-bar.cc

@@ -170,6 +170,14 @@ public:
                 name, sub);
         }
 
+        if (type == actPostBuildHook) {
+            auto name = storePathToName(getS(fields, 0));
+            if (hasSuffix(name, ".drv"))
+                name.resize(name.size() - 4);
+            i->s = fmt("post-build " ANSI_BOLD "%s" ANSI_NORMAL, name);
+            i->name = DrvName(name).name;
+        }
+
         if (type == actQueryPathInfo) {
             auto name = storePathToName(getS(fields, 0));
             i->s = fmt("querying " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1));
@@ -228,14 +236,18 @@ public:
             update(*state);
         }
 
-        else if (type == resBuildLogLine) {
+        else if (type == resBuildLogLine || type == resPostBuildLogLine) {
             auto lastLine = trim(getS(fields, 0));
             if (!lastLine.empty()) {
                 auto i = state->its.find(act);
                 assert(i != state->its.end());
                 ActInfo info = *i->second;
                 if (printBuildLogs) {
-                    log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + "> " + ANSI_NORMAL + lastLine);
+                    auto suffix = "> ";
+                    if (type == resPostBuildLogLine) {
+                        suffix = " (post)> ";
+                    }
+                    log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + suffix + ANSI_NORMAL + lastLine);
                 } else {
                     state->activities.erase(i->second);
                     info.lastLine = lastLine;

tests/dependencies.nix

@@ -17,6 +17,7 @@ let {
     builder = ./dependencies.builder0.sh + "/FOOBAR/../.";
     input1 = input1 + "/.";
     input2 = "${input2}/.";
+    input1_drv = input1;
     meta.description = "Random test package";
   };
 

tests/local.mk

@@ -26,7 +26,8 @@ nix_tests = \
   check.sh \
   plugins.sh \
   search.sh \
-  nix-copy-ssh.sh
+  nix-copy-ssh.sh \
+  post-hook.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))

tests/post-hook.sh

@@ -0,0 +1,15 @@
+source common.sh
+
+clearStore
+
+export REMOTE_STORE=$TEST_ROOT/remote_store
+
+# Build the dependencies and push them to the remote store
+nix-build dependencies.nix --post-build-hook $PWD/push_to_store.sh
+
+clearStore
+
+# Ensure that we the remote store contains both the runtime and buildtime
+# closure of what we've just built
+nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix
+nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix input1_drv

tests/push_to_store.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+echo Pushing "$@" to "$REMOTE_STORE"
+echo -n "$OUT_PATHS" | xargs -d: nix copy --to "$REMOTE_STORE" --no-require-sigs