Add a mechanism for derivation attributes to reference the derivation's outputs

For example, you can now say:

  configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"}";

The strings returned by the ‘placeholder’ builtin are replaced at
build time by the actual store paths corresponding to the specified
outputs.

Previously, you had to work around the inability to self-reference by doing stuff like:

  preConfigure = ''
    configureFlags+=" --prefix $out --includedir=$dev"
  '';

or rely on ad-hoc variable interpolation semantics in Autoconf or Make
(e.g. --prefix=\$(out)), which doesn't always work.

src/libexpr/primops.cc

@@ -673,6 +673,19 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 }
 
 
+/* Return a placeholder string for the specified output that will be
+   substituted by the corresponding output path at build time. For
+   example, ‘placeholder "out"’ returns the string
+   /1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9. At build
+   time, any occurence of this string in an derivation attribute will
+   be replaced with the concrete path in the Nix store of the output
+   ‘out’. */
+static void prim_placeholder(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    mkString(v, hashPlaceholder(state.forceStringNoCtx(*args[0], pos)));
+}
+
+
 /*************************************************************
  * Paths
  *************************************************************/
@@ -1893,6 +1906,7 @@ void EvalState::createBaseEnv()
 
     // Derivations
     addPrimOp("derivationStrict", 1, prim_derivationStrict);
+    addPrimOp("placeholder", 1, prim_placeholder);
 
     // Networking
     addPrimOp("__fetchurl", 1, prim_fetchurl);

src/libstore/build.cc

@@ -652,18 +652,15 @@ HookInstance::~HookInstance()
 //////////////////////////////////////////////////////////////////////
 
 
-typedef map<string, string> HashRewrites;
+typedef map<std::string, std::string> StringRewrites;
 
 
-string rewriteHashes(string s, const HashRewrites & rewrites)
+std::string rewriteStrings(std::string s, const StringRewrites & rewrites)
 {
     for (auto & i : rewrites) {
-        assert(i.first.size() == i.second.size());
         size_t j = 0;
-        while ((j = s.find(i.first, j)) != string::npos) {
-            debug(format("rewriting @ %1%") % j);
-            s.replace(j, i.second.size(), i.second);
-        }
+        while ((j = s.find(i.first, j)) != string::npos)
+            s.replace(j, i.first.size(), i.second);
     }
     return s;
 }
@@ -782,7 +779,7 @@ private:
 #endif
 
     /* Hash rewriting. */
-    HashRewrites rewritesToTmp, rewritesFromTmp;
+    StringRewrites inputRewrites, outputRewrites;
     typedef map<Path, Path> RedirectedOutputs;
     RedirectedOutputs redirectedOutputs;
 
@@ -1774,6 +1771,10 @@ void DerivationGoal::startBuilder()
         for (auto & i : varNames) env[i] = getEnv(i);
     }
 
+    /* Substitute output placeholders with the actual output paths. */
+    for (auto & output : drv->outputs)
+        inputRewrites[hashPlaceholder(output.first)] = output.second.path;
+
     /* The `exportReferencesGraph' feature allows the references graph
        to be passed to a builder.  This attribute should be a list of
        pairs [name1 path1 name2 path2 ...].  The references graph of
@@ -2418,7 +2419,7 @@ void DerivationGoal::runChild()
         /* Fill in the environment. */
         Strings envStrs;
         for (auto & i : env)
-            envStrs.push_back(rewriteHashes(i.first + "=" + i.second, rewritesToTmp));
+            envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites));
 
         /* If we are running in `build-users' mode, then switch to the
            user we allocated above.  Make sure that we drop all root
@@ -2560,7 +2561,7 @@ void DerivationGoal::runChild()
         }
 
         for (auto & i : drv->args)
-            args.push_back(rewriteHashes(i, rewritesToTmp));
+            args.push_back(rewriteStrings(i, inputRewrites));
 
         restoreSIGPIPE();
 
@@ -2682,7 +2683,7 @@ void DerivationGoal::registerOutputs()
 
         /* Apply hash rewriting if necessary. */
         bool rewritten = false;
-        if (!rewritesFromTmp.empty()) {
+        if (!outputRewrites.empty()) {
             printMsg(lvlError, format("warning: rewriting hashes in ‘%1%’; cross fingers") % path);
 
             /* Canonicalise first.  This ensures that the path we're
@@ -2694,7 +2695,7 @@ void DerivationGoal::registerOutputs()
             StringSink sink;
             dumpPath(actualPath, sink);
             deletePath(actualPath);
-            sink.s = make_ref<std::string>(rewriteHashes(*sink.s, rewritesFromTmp));
+            sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites));
             StringSource source(*sink.s);
             restorePath(actualPath, source);
 
@@ -3033,8 +3034,8 @@ Path DerivationGoal::addHashRewrite(const Path & path)
     Path p = worker.store.storeDir + "/" + h2 + string(path, worker.store.storeDir.size() + 33);
     deletePath(p);
     assert(path.size() == p.size());
-    rewritesToTmp[h1] = h2;
-    rewritesFromTmp[h2] = h1;
+    inputRewrites[h1] = h2;
+    outputRewrites[h2] = h1;
     redirectedOutputs[path] = p;
     return p;
 }

src/libstore/derivations.cc

@@ -390,4 +390,11 @@ Sink & operator << (Sink & out, const BasicDerivation & drv)
 }
 
 
+std::string hashPlaceholder(const std::string & outputName)
+{
+    // FIXME: memoize?
+    return "/" + printHash32(hashString(htSHA256, "nix-output:" + outputName));
+}
+
+
 }

src/libstore/derivations.hh

@@ -117,4 +117,6 @@ struct Sink;
 Source & readDerivation(Source & in, Store & store, BasicDerivation & drv);
 Sink & operator << (Sink & out, const BasicDerivation & drv);
 
+std::string hashPlaceholder(const std::string & outputName);
+
 }

tests/config.nix

@@ -13,7 +13,7 @@ rec {
     derivation ({
       inherit system;
       builder = shell;
-      args = ["-e" args.builder];
+      args = ["-e" args.builder or (builtins.toFile "builder.sh" "eval \"$buildCommand\"")];
       PATH = path;
     } // removeAttrs args ["builder" "meta"])
     // { meta = args.meta or {}; };

tests/local.mk

@@ -10,7 +10,8 @@ nix_tests = \
   timeout.sh secure-drv-outputs.sh nix-channel.sh \
   multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \
   binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh \
-  check-reqs.sh pass-as-file.sh tarball.sh restricted.sh
+  check-reqs.sh pass-as-file.sh tarball.sh restricted.sh \
+  placeholders.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))

tests/placeholders.sh

@@ -0,0 +1,22 @@
+source common.sh
+
+clearStore
+
+nix-build --no-out-link -E '
+  with import ./config.nix;
+
+  mkDerivation {
+    name = "placeholders";
+    outputs = [ "out" "bin" "dev" ];
+    buildCommand = "
+      echo foo1 > $out
+      echo foo2 > $bin
+      echo foo3 > $dev
+      [[ $(cat ${placeholder "out"}) = foo1 ]]
+      [[ $(cat ${placeholder "bin"}) = foo2 ]]
+      [[ $(cat ${placeholder "dev"}) = foo3 ]]
+    ";
+  }
+'
+
+echo XYZZY