Merge remote-tracking branch 'upstream/master' into trustless-remote-builder-simple

configure.ac

@@ -184,7 +184,7 @@ fi
 
 # Look for OpenSSL, a required dependency. FIXME: this is only (maybe)
 # used by S3BinaryCacheStore.
-PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"])
+PKG_CHECK_MODULES([OPENSSL], [libcrypto >= 1.1.1], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"])
 
 
 # Look for libarchive.

doc/manual/generate-manpage.nix

@@ -10,7 +10,9 @@ let
 
       result = ''
         > **Warning** \
-        > This program is **experimental** and its interface is subject to change.
+        > This program is
+        > [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command)
+        > and its interface is subject to change.
 
         # Name
 
@@ -29,19 +31,18 @@ let
 
       showSynopsis = command: args:
         let
-          showArgument = arg: "*${arg.label}*" + (if arg ? arity then "" else "...");
+          showArgument = arg: "*${arg.label}*" + optionalString (! arg ? arity) "...";
           arguments = concatStringsSep " " (map showArgument args);
         in ''
          `${command}` [*option*...] ${arguments}
         '';
 
-      maybeSubcommands = if details ? commands && details.commands != {}
+      maybeSubcommands = optionalString (details ? commands && details.commands != {})
-        then ''
+         ''
            where *subcommand* is one of the following:
 
            ${subcommands}
-         ''
+         '';
-        else "";
 
       subcommands = if length categories > 1
         then listCategories
@@ -63,12 +64,11 @@ let
         * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description}
       '';
 
-      maybeDocumentation =
+      maybeDocumentation = optionalString
-        if details ? doc
+        (details ? doc)
-        then replaceStrings ["@stores@"] [storeDocs] details.doc
+        (replaceStrings ["@stores@"] [storeDocs] details.doc);
-        else "";
 
-      maybeOptions = if details.flags == {} then "" else ''
+      maybeOptions = optionalString (details.flags != {}) ''
         # Options
 
         ${showOptions details.flags toplevel.flags}
@@ -78,15 +78,19 @@ let
         let
           allOptions = options // commonOptions;
           showCategory = cat: ''
-            ${if cat != "" then "**${cat}:**" else ""}
+            ${optionalString (cat != "") "**${cat}:**"}
 
             ${listOptions (filterAttrs (n: v: v.category == cat) allOptions)}
             '';
           listOptions = opts: concatStringsSep "\n" (attrValues (mapAttrs showOption opts));
           showOption = name: option:
             let
-              shortName = if option ? shortName then "/ `-${option.shortName}`" else "";
+              shortName = optionalString
-              labels = if option ? labels then (concatStringsSep " " (map (s: "*${s}*") option.labels)) else "";
+                (option ? shortName)
+                ("/ `-${option.shortName}`");
+              labels = optionalString
+                (option ? labels)
+                (concatStringsSep " " (map (s: "*${s}*") option.labels));
             in trim ''
               - `--${name}` ${shortName} ${labels}
 

doc/manual/src/command-ref/experimental-commands.md

@@ -1,6 +1,6 @@
 # Experimental Commands
 
-This section lists experimental commands.
+This section lists [experimental commands](@docroot@/contributing/experimental-features.md#xp-feature-nix-command).
 
 > **Warning**
 >

doc/manual/src/glossary.md

@@ -225,3 +225,9 @@
     [string]: ./language/values.md#type-string
     [path]: ./language/values.md#type-path
     [attribute name]: ./language/values.md#attribute-set
+
+  - [experimental feature]{#gloss-experimental-feature}\
+    Not yet stabilized functionality guarded by named experimental feature flags.
+    These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting.
+
+    See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md).

doc/manual/src/language/advanced-attributes.md

@@ -208,12 +208,26 @@ Derivations can declare some infrequently used optional attributes.
     about converting to and from base-32 notation.)
 
   - [`__contentAddressed`]{#adv-attr-__contentAddressed}
-    If this **experimental** attribute is set to true, then the derivation
+    > **Warning**
+    > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md).
+    >
+    > To use this attribute, you must enable the
+    > [`ca-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) experimental feature.
+    > For example, in [nix.conf](../command-ref/conf-file.md) you could add:
+    >
+    > ```
+    > extra-experimental-features = ca-derivations
+    > ```
+
+    If this attribute is set to `true`, then the derivation
     outputs will be stored in a content-addressed location rather than the
     traditional input-addressed one.
-    This only has an effect if the `ca-derivations` experimental feature is enabled.
 
-    Setting this attribute also requires setting `outputHashMode` and `outputHashAlgo` like for *fixed-output derivations* (see above).
+    Setting this attribute also requires setting
+    [`outputHashMode`](#adv-attr-outputHashMode)
+    and
+    [`outputHashAlgo`](#adv-attr-outputHashAlgo)
+    like for *fixed-output derivations* (see above).
 
   - [`passAsFile`]{#adv-attr-passAsFile}\
     A list of names of attributes that should be passed via files rather
@@ -307,9 +321,11 @@ Derivations can declare some infrequently used optional attributes.
 
   - [`unsafeDiscardReferences`]{#adv-attr-unsafeDiscardReferences}\
     > **Warning**
-    > This is an experimental feature.
+    > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md).
     >
-    > To enable it, add the following to [nix.conf](../command-ref/conf-file.md):
+    > To use this attribute, you must enable the
+    > [`discard-references`](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) experimental feature.
+    > For example, in [nix.conf](../command-ref/conf-file.md) you could add:
     >
     > ```
     > extra-experimental-features = discard-references

doc/manual/src/quick-start.md

@@ -19,7 +19,7 @@ to subsequent chapters.
    channel:
 
    ```console
-   $ nix-env -qaP
+   $ nix-env --query --available --attr-path
    nixpkgs.docbook_xml_dtd_43                    docbook-xml-4.3
    nixpkgs.docbook_xml_dtd_45                    docbook-xml-4.5
    nixpkgs.firefox                               firefox-33.0.2
@@ -31,7 +31,7 @@ to subsequent chapters.
 1. Install some packages from the channel:
 
    ```console
-   $ nix-env -iA nixpkgs.hello
+   $ nix-env --install --attr nixpkgs.hello
    ```
 
    This should download pre-built packages; it should not build them
@@ -49,13 +49,13 @@ to subsequent chapters.
 1. Uninstall a package:
 
    ```console
-   $ nix-env -e hello
+   $ nix-env --uninstall hello
    ```
 
 1. You can also test a package without installing it:
 
    ```console
-   $ nix-shell -p hello
+   $ nix-shell --packages hello
    ```
 
    This builds or downloads GNU Hello and its dependencies, then drops
@@ -76,7 +76,7 @@ to subsequent chapters.
 
    ```console
    $ nix-channel --update nixpkgs
-   $ nix-env -u '*'
+   $ nix-env --upgrade '*'
    ```
 
    The latter command will upgrade each installed package for which
@@ -95,5 +95,5 @@ to subsequent chapters.
    them:
 
    ```console
-   $ nix-collect-garbage -d
+   $ nix-collect-garbage --delete-old
    ```

doc/manual/utils.nix

@@ -42,7 +42,9 @@ rec {
   filterAttrs = pred: set:
     listToAttrs (concatMap (name: let v = set.${name}; in if pred name v then [(nameValuePair name v)] else []) (attrNames set));
 
-  showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value }:
+  optionalString = cond: string: if cond then string else "";
+
+  showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value, experimentalFeature }:
     let
       result = squash ''
           - ${if useAnchors
@@ -52,10 +54,28 @@ rec {
           ${indent "  " body}
         '';
 
+      experimentalFeatureNote = optionalString (experimentalFeature != null) ''
+          > **Warning**
+          > This setting is part of an
+          > [experimental feature](@docroot@/contributing/experimental-features.md).
+
+          To change this setting, you need to make sure the corresponding experimental feature,
+          [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}),
+          is enabled.
+          For example, include the following in [`nix.conf`](#):
+
+          ```
+          extra-experimental-features = ${experimentalFeature}
+          ${name} = ...
+          ```
+        '';
+
       # separate body to cleanly handle indentation
       body = ''
           ${description}
 
+          ${experimentalFeatureNote}
+
           **Default:** ${showDefault documentDefault defaultValue}
 
           ${showAliases aliases}
@@ -74,7 +94,7 @@ rec {
         else "*machine-specific*";
 
       showAliases = aliases:
-          if aliases == [] then "" else
+          optionalString (aliases != [])
             "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}";
 
     in result;

perl/lib/Nix/Store.xs

@@ -27,8 +27,6 @@ static ref<Store> store()
     if (!_store) {
         try {
             initLibStore();
-            loadConfFile();
-            settings.lockCPU = false;
             _store = openStore();
         } catch (Error & e) {
             croak("%s", e.what());

src/libcmd/repl.cc

@@ -40,6 +40,7 @@ extern "C" {
 #include "markdown.hh"
 #include "local-fs-store.hh"
 #include "progress-bar.hh"
+#include "print.hh"
 
 #if HAVE_BOEHMGC
 #define GC_INCLUDE_NEW
@@ -425,6 +426,7 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
 }
 
 
+// FIXME: DRY and match or use the parser
 static bool isVarName(std::string_view s)
 {
     if (s.size() == 0) return false;
@@ -894,17 +896,6 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
 }
 
 
-std::ostream & printStringValue(std::ostream & str, const char * string) {
-    str << "\"";
-    for (const char * i = string; *i; i++)
-        if (*i == '\"' || *i == '\\') str << "\\" << *i;
-        else if (*i == '\n') str << "\\n";
-        else if (*i == '\r') str << "\\r";
-        else if (*i == '\t') str << "\\t";
-        else str << *i;
-    str << "\"";
-    return str;
-}
 
 
 // FIXME: lot of cut&paste from Nix's eval.cc.
@@ -922,12 +913,14 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
         break;
 
     case nBool:
-        str << ANSI_CYAN << (v.boolean ? "true" : "false") << ANSI_NORMAL;
+        str << ANSI_CYAN;
+        printLiteralBool(str, v.boolean);
+        str << ANSI_NORMAL;
         break;
 
     case nString:
         str << ANSI_WARNING;
-        printStringValue(str, v.string.s);
+        printLiteralString(str, v.string.s);
         str << ANSI_NORMAL;
         break;
 
@@ -964,10 +957,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
                 sorted.emplace(state->symbols[i.name], i.value);
 
             for (auto & i : sorted) {
-                if (isVarName(i.first))
+                printAttributeName(str, i.first);
-                    str << i.first;
-                else
-                    printStringValue(str, i.first.c_str());
                 str << " = ";
                 if (seen.count(i.second))
                     str << "«repeated»";

src/libexpr/eval.cc

@@ -9,6 +9,7 @@
 #include "filetransfer.hh"
 #include "function-trace.hh"
 #include "profiles.hh"
+#include "print.hh"
 
 #include <algorithm>
 #include <chrono>
@@ -104,18 +105,10 @@ void Value::print(const SymbolTable & symbols, std::ostream & str,
         str << integer;
         break;
     case tBool:
-        str << (boolean ? "true" : "false");
+        printLiteralBool(str, boolean);
         break;
     case tString:
-        str << "\"";
+        printLiteralString(str, string.s);
-        for (const char * i = string.s; *i; i++)
-            if (*i == '\"' || *i == '\\') str << "\\" << *i;
-            else if (*i == '\n') str << "\\n";
-            else if (*i == '\r') str << "\\r";
-            else if (*i == '\t') str << "\\t";
-            else if (*i == '$' && *(i+1) == '{') str << "\\" << *i;
-            else str << *i;
-        str << "\"";
         break;
     case tPath:
         str << path; // !!! escaping?

src/libexpr/nixexpr.cc

@@ -3,6 +3,7 @@
 #include "eval.hh"
 #include "symbol-table.hh"
 #include "util.hh"
+#include "print.hh"
 
 #include <cstdlib>
 
@@ -60,45 +61,12 @@ Pos::operator std::shared_ptr<AbstractPos>() const
     return pos;
 }
 
-/* Displaying abstract syntax trees. */
+// FIXME: remove, because *symbols* are abstract and do not have a single
-
+//        textual representation; see printIdentifier()
-static void showString(std::ostream & str, std::string_view s)
-{
-    str << '"';
-    for (auto c : s)
-        if (c == '"' || c == '\\' || c == '$') str << "\\" << c;
-        else if (c == '\n') str << "\\n";
-        else if (c == '\r') str << "\\r";
-        else if (c == '\t') str << "\\t";
-        else str << c;
-    str << '"';
-}
-
 std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol)
 {
     std::string_view s = symbol;
-
+    return printIdentifier(str, s);
-    if (s.empty())
-        str << "\"\"";
-    else if (s == "if") // FIXME: handle other keywords
-        str << '"' << s << '"';
-    else {
-        char c = s[0];
-        if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) {
-            showString(str, s);
-            return str;
-        }
-        for (auto c : s)
-            if (!((c >= 'a' && c <= 'z') ||
-                  (c >= 'A' && c <= 'Z') ||
-                  (c >= '0' && c <= '9') ||
-                  c == '_' || c == '\'' || c == '-')) {
-                showString(str, s);
-                return str;
-            }
-        str << s;
-    }
-    return str;
 }
 
 void Expr::show(const SymbolTable & symbols, std::ostream & str) const
@@ -118,7 +86,7 @@ void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const
 
 void ExprString::show(const SymbolTable & symbols, std::ostream & str) const
 {
-    showString(str, s);
+    printLiteralString(str, s);
 }
 
 void ExprPath::show(const SymbolTable & symbols, std::ostream & str) const

src/libexpr/print.cc

@@ -0,0 +1,78 @@
+#include "print.hh"
+
+namespace nix {
+
+std::ostream &
+printLiteralString(std::ostream & str, const std::string_view string)
+{
+    str << "\"";
+    for (auto i = string.begin(); i != string.end(); ++i) {
+        if (*i == '\"' || *i == '\\') str << "\\" << *i;
+        else if (*i == '\n') str << "\\n";
+        else if (*i == '\r') str << "\\r";
+        else if (*i == '\t') str << "\\t";
+        else if (*i == '$' && *(i+1) == '{') str << "\\" << *i;
+        else str << *i;
+    }
+    str << "\"";
+    return str;
+}
+
+std::ostream &
+printLiteralBool(std::ostream & str, bool boolean)
+{
+    str << (boolean ? "true" : "false");
+    return str;
+}
+
+std::ostream &
+printIdentifier(std::ostream & str, std::string_view s) {
+    if (s.empty())
+        str << "\"\"";
+    else if (s == "if") // FIXME: handle other keywords
+        str << '"' << s << '"';
+    else {
+        char c = s[0];
+        if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) {
+            printLiteralString(str, s);
+            return str;
+        }
+        for (auto c : s)
+            if (!((c >= 'a' && c <= 'z') ||
+                  (c >= 'A' && c <= 'Z') ||
+                  (c >= '0' && c <= '9') ||
+                  c == '_' || c == '\'' || c == '-')) {
+                printLiteralString(str, s);
+                return str;
+            }
+        str << s;
+    }
+    return str;
+}
+
+// FIXME: keywords
+static bool isVarName(std::string_view s)
+{
+    if (s.size() == 0) return false;
+    char c = s[0];
+    if ((c >= '0' && c <= '9') || c == '-' || c == '\'') return false;
+    for (auto & i : s)
+        if (!((i >= 'a' && i <= 'z') ||
+              (i >= 'A' && i <= 'Z') ||
+              (i >= '0' && i <= '9') ||
+              i == '_' || i == '-' || i == '\''))
+            return false;
+    return true;
+}
+
+std::ostream &
+printAttributeName(std::ostream & str, std::string_view name) {
+    if (isVarName(name))
+        str << name;
+    else
+        printLiteralString(str, name);
+    return str;
+}
+
+
+}

src/libexpr/print.hh

@@ -0,0 +1,48 @@
+#pragma once
+/**
+ * @file
+ * @brief Common printing functions for the Nix language
+ *
+ * While most types come with their own methods for printing, they share some
+ * functions that are placed here.
+ */
+
+#include <iostream>
+
+namespace nix {
+    /**
+     * Print a string as a Nix string literal.
+     *
+     * Quotes and fairly minimal escaping are added.
+     *
+     * @param s The logical string
+     */
+    std::ostream & printLiteralString(std::ostream & o, std::string_view s);
+    inline std::ostream & printLiteralString(std::ostream & o, const char * s) {
+        return printLiteralString(o, std::string_view(s));
+    }
+    inline std::ostream & printLiteralString(std::ostream & o, const std::string & s) {
+        return printLiteralString(o, std::string_view(s));
+    }
+
+    /** Print `true` or `false`. */
+    std::ostream & printLiteralBool(std::ostream & o, bool b);
+
+    /**
+     * Print a string as an attribute name in the Nix expression language syntax.
+     *
+     * Prints a quoted string if necessary.
+     */
+    std::ostream & printAttributeName(std::ostream & o, std::string_view s);
+
+    /**
+     * Print a string as an identifier in the Nix expression language syntax.
+     *
+     * FIXME: "identifier" is ambiguous. Identifiers do not have a single
+     *        textual representation. They can be used in variable references,
+     *        let bindings, left-hand sides or attribute names in a select
+     *        expression, or something else entirely, like JSON. Use one of the
+     *        `print*` functions instead.
+     */
+    std::ostream & printIdentifier(std::ostream & o, std::string_view s);
+}

src/libmain/shared.cc

@@ -10,7 +10,6 @@
 #include <cctype>
 #include <exception>
 #include <iostream>
-#include <mutex>
 
 #include <cstdlib>
 #include <sys/time.h>
@@ -20,16 +19,9 @@
 #ifdef __linux__
 #include <features.h>
 #endif
-#ifdef __GLIBC__
-#include <gnu/lib-names.h>
-#include <nss.h>
-#include <dlfcn.h>
-#endif
 
 #include <openssl/crypto.h>
 
-#include <sodium.h>
-
 
 namespace nix {
 
@@ -115,57 +107,6 @@ std::string getArg(const std::string & opt,
     return *i;
 }
 
-
-#if OPENSSL_VERSION_NUMBER < 0x10101000L
-/* OpenSSL is not thread-safe by default - it will randomly crash
-   unless the user supplies a mutex locking function. So let's do
-   that. */
-static std::vector<std::mutex> opensslLocks;
-
-static void opensslLockCallback(int mode, int type, const char * file, int line)
-{
-    if (mode & CRYPTO_LOCK)
-        opensslLocks[type].lock();
-    else
-        opensslLocks[type].unlock();
-}
-#endif
-
-static std::once_flag dns_resolve_flag;
-
-static void preloadNSS() {
-    /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
-       one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
-       been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
-       load its lookup libraries in the parent before any child gets a chance to. */
-    std::call_once(dns_resolve_flag, []() {
-#ifdef __GLIBC__
-        /* On linux, glibc will run every lookup through the nss layer.
-         * That means every lookup goes, by default, through nscd, which acts as a local
-         * cache.
-         * Because we run builds in a sandbox, we also remove access to nscd otherwise
-         * lookups would leak into the sandbox.
-         *
-         * But now we have a new problem, we need to make sure the nss_dns backend that
-         * does the dns lookups when nscd is not available is loaded or available.
-         *
-         * We can't make it available without leaking nix's environment, so instead we'll
-         * load the backend, and configure nss so it does not try to run dns lookups
-         * through nscd.
-         *
-         * This is technically only used for builtins:fetch* functions so we only care
-         * about dns.
-         *
-         * All other platforms are unaffected.
-         */
-        if (!dlopen(LIBNSS_DNS_SO, RTLD_NOW))
-            warn("unable to load nss_dns backend");
-        // FIXME: get hosts entry from nsswitch.conf.
-        __nss_configure_lookup("hosts", "files dns");
-#endif
-    });
-}
-
 static void sigHandler(int signo) { }
 
 
@@ -177,16 +118,7 @@ void initNix()
     std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf));
 #endif
 
-#if OPENSSL_VERSION_NUMBER < 0x10101000L
+    initLibStore();
-    /* Initialise OpenSSL locking. */
-    opensslLocks = std::vector<std::mutex>(CRYPTO_num_locks());
-    CRYPTO_set_locking_callback(opensslLockCallback);
-#endif
-
-    if (sodium_init() == -1)
-        throw Error("could not initialise libsodium");
-
-    loadConfFile();
 
     startSignalHandlerThread();
 
@@ -223,7 +155,10 @@ void initNix()
     if (sigaction(SIGTRAP, &act, 0)) throw SysError("handling SIGTRAP");
 #endif
 
-    /* Register a SIGSEGV handler to detect stack overflows. */
+    /* Register a SIGSEGV handler to detect stack overflows.
+       Why not initLibExpr()? initGC() is essentially that, but
+       detectStackOverflow is not an instance of the init function concept, as
+       it may have to be invoked more than once per process. */
     detectStackOverflow();
 
     /* There is no privacy in the Nix system ;-)  At least not for
@@ -236,16 +171,6 @@ void initNix()
     gettimeofday(&tv, 0);
     srandom(tv.tv_usec);
 
-    /* On macOS, don't use the per-session TMPDIR (as set e.g. by
-       sshd). This breaks build users because they don't have access
-       to the TMPDIR, in particular in ‘nix-store --serve’. */
-#if __APPLE__
-    if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/"))
-        unsetenv("TMPDIR");
-#endif
-
-    preloadNSS();
-    initLibStore();
 }
 
 

src/libstore/derivations.cc

@@ -313,6 +313,15 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi
 }
 
 
+/**
+ * Print a derivation string literal to an `std::string`.
+ *
+ * This syntax does not generalize to the expression language, which needs to
+ * escape `$`.
+ *
+ * @param res Where to print to
+ * @param s Which logical string to print
+ */
 static void printString(std::string & res, std::string_view s)
 {
     boost::container::small_vector<char, 64 * 1024> buffer;

src/libstore/derived-path.cc

@@ -62,15 +62,31 @@ std::string DerivedPath::Opaque::to_string(const Store & store) const
 std::string DerivedPath::Built::to_string(const Store & store) const
 {
     return store.printStorePath(drvPath)
-        + "!"
+        + '^'
+        + outputs.to_string();
+}
+
+std::string DerivedPath::Built::to_string_legacy(const Store & store) const
+{
+    return store.printStorePath(drvPath)
+        + '!'
         + outputs.to_string();
 }
 
 std::string DerivedPath::to_string(const Store & store) const
 {
-    return std::visit(
+    return std::visit(overloaded {
-        [&](const auto & req) { return req.to_string(store); },
+        [&](const DerivedPath::Built & req) { return req.to_string(store); },
-        this->raw());
+        [&](const DerivedPath::Opaque & req) { return req.to_string(store); },
+    }, this->raw());
+}
+
+std::string DerivedPath::to_string_legacy(const Store & store) const
+{
+    return std::visit(overloaded {
+        [&](const DerivedPath::Built & req) { return req.to_string_legacy(store); },
+        [&](const DerivedPath::Opaque & req) { return req.to_string(store); },
+    }, this->raw());
 }
 
 
@@ -87,14 +103,24 @@ DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_vi
     };
 }
 
-DerivedPath DerivedPath::parse(const Store & store, std::string_view s)
+static inline DerivedPath parseWith(const Store & store, std::string_view s, std::string_view separator)
 {
-    size_t n = s.find("!");
+    size_t n = s.find(separator);
     return n == s.npos
         ? (DerivedPath) DerivedPath::Opaque::parse(store, s)
         : (DerivedPath) DerivedPath::Built::parse(store, s.substr(0, n), s.substr(n + 1));
 }
 
+DerivedPath DerivedPath::parse(const Store & store, std::string_view s)
+{
+	return parseWith(store, s, "^");
+}
+
+DerivedPath DerivedPath::parseLegacy(const Store & store, std::string_view s)
+{
+	return parseWith(store, s, "!");
+}
+
 RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
 {
     RealisedPath::Set res;

src/libstore/derived-path.hh

@@ -48,8 +48,18 @@ struct DerivedPathBuilt {
     StorePath drvPath;
     OutputsSpec outputs;
 
+    /**
+     * Uses `^` as the separator
+     */
     std::string to_string(const Store & store) const;
-    static DerivedPathBuilt parse(const Store & store, std::string_view, std::string_view);
+    /**
+     * Uses `!` as the separator
+     */
+    std::string to_string_legacy(const Store & store) const;
+    /**
+     * The caller splits on the separator, so it works for both variants.
+     */
+    static DerivedPathBuilt parse(const Store & store, std::string_view drvPath, std::string_view outputs);
     nlohmann::json toJSON(ref<Store> store) const;
 
     GENERATE_CMP(DerivedPathBuilt, me->drvPath, me->outputs);
@@ -81,8 +91,22 @@ struct DerivedPath : _DerivedPathRaw {
         return static_cast<const Raw &>(*this);
     }
 
+    /**
+     * Uses `^` as the separator
+     */
     std::string to_string(const Store & store) const;
+    /**
+     * Uses `!` as the separator
+     */
+    std::string to_string_legacy(const Store & store) const;
+    /**
+     * Uses `^` as the separator
+     */
     static DerivedPath parse(const Store & store, std::string_view);
+    /**
+     * Uses `!` as the separator
+     */
+    static DerivedPath parseLegacy(const Store & store, std::string_view);
 };
 
 /**

src/libstore/dummy-store.cc

@@ -71,6 +71,9 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store
     void queryRealisationUncached(const DrvOutput &,
         Callback<std::shared_ptr<const Realisation>> callback) noexcept override
     { callback(nullptr); }
+
+    virtual ref<FSAccessor> getFSAccessor() override
+    { unsupported("getFSAccessor"); }
 };
 
 static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regDummyStore;

src/libstore/globals.cc

@@ -7,12 +7,20 @@
 
 #include <algorithm>
 #include <map>
+#include <mutex>
 #include <thread>
 #include <dlfcn.h>
 #include <sys/utsname.h>
 
 #include <nlohmann/json.hpp>
 
+#include <sodium/core.h>
+
+#ifdef __GLIBC__
+#include <gnu/lib-names.h>
+#include <nss.h>
+#include <dlfcn.h>
+#endif
 
 namespace nix {
 
@@ -41,7 +49,6 @@ Settings::Settings()
     , nixDaemonSocketFile(canonPath(getEnvNonEmpty("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH)))
 {
     buildUsersGroup = getuid() == 0 ? "nixbld" : "";
-    lockCPU = getEnv("NIX_AFFINITY_HACK") == "1";
     allowSymlinkedStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1";
 
     auto sslOverride = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or(""));
@@ -281,6 +288,42 @@ void initPlugins()
     settings.pluginFiles.pluginsLoaded = true;
 }
 
+static void preloadNSS()
+{
+    /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
+       one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
+       been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
+       load its lookup libraries in the parent before any child gets a chance to. */
+    static std::once_flag dns_resolve_flag;
+
+    std::call_once(dns_resolve_flag, []() {
+#ifdef __GLIBC__
+        /* On linux, glibc will run every lookup through the nss layer.
+         * That means every lookup goes, by default, through nscd, which acts as a local
+         * cache.
+         * Because we run builds in a sandbox, we also remove access to nscd otherwise
+         * lookups would leak into the sandbox.
+         *
+         * But now we have a new problem, we need to make sure the nss_dns backend that
+         * does the dns lookups when nscd is not available is loaded or available.
+         *
+         * We can't make it available without leaking nix's environment, so instead we'll
+         * load the backend, and configure nss so it does not try to run dns lookups
+         * through nscd.
+         *
+         * This is technically only used for builtins:fetch* functions so we only care
+         * about dns.
+         *
+         * All other platforms are unaffected.
+         */
+        if (!dlopen(LIBNSS_DNS_SO, RTLD_NOW))
+            warn("unable to load nss_dns backend");
+        // FIXME: get hosts entry from nsswitch.conf.
+        __nss_configure_lookup("hosts", "files dns");
+#endif
+    });
+}
+
 static bool initLibStoreDone = false;
 
 void assertLibStoreInitialized() {
@@ -291,6 +334,24 @@ void assertLibStoreInitialized() {
 }
 
 void initLibStore() {
+
+    initLibUtil();
+
+    if (sodium_init() == -1)
+        throw Error("could not initialise libsodium");
+
+    loadConfFile();
+
+    preloadNSS();
+
+    /* On macOS, don't use the per-session TMPDIR (as set e.g. by
+       sshd). This breaks build users because they don't have access
+       to the TMPDIR, in particular in ‘nix-store --serve’. */
+#if __APPLE__
+    if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/"))
+        unsetenv("TMPDIR");
+#endif
+
     initLibStoreDone = true;
 }
 

src/libstore/globals.hh

@@ -328,16 +328,6 @@ public:
           users in `build-users-group`.
 
           UIDs are allocated starting at 872415232 (0x34000000) on Linux and 56930 on macOS.
-
-          > **Warning**
-          > This is an experimental feature.
-
-          To enable it, add the following to [`nix.conf`](#):
-
-          ```
-          extra-experimental-features = auto-allocate-uids
-          auto-allocate-uids = true
-          ```
         )"};
 
     Setting<uint32_t> startId{this,
@@ -367,16 +357,6 @@ public:
 
           Cgroups are required and enabled automatically for derivations
           that require the `uid-range` system feature.
-
-          > **Warning**
-          > This is an experimental feature.
-
-          To enable it, add the following to [`nix.conf`](#):
-
-          ```
-          extra-experimental-features = cgroups
-          use-cgroups = true
-          ```
         )"};
     #endif
 
@@ -478,11 +458,6 @@ public:
         )",
         {"env-keep-derivations"}};
 
-    /**
-     * Whether to lock the Nix client and worker to the same CPU.
-     */
-    bool lockCPU;
-
     Setting<SandboxMode> sandboxMode{
         this,
         #if __linux__

src/libstore/legacy-ssh-store.cc

@@ -342,6 +342,9 @@ public:
     void ensurePath(const StorePath & path) override
     { unsupported("ensurePath"); }
 
+    virtual ref<FSAccessor> getFSAccessor() override
+    { unsupported("getFSAccessor"); }
+
     void computeFSClosure(const StorePathSet & paths,
         StorePathSet & out, bool flipDirection = false,
         bool includeOutputs = false, bool includeDerivers = false) override

src/libstore/remote-store.cc

@@ -90,12 +90,12 @@ void write(const Store & store, Sink & out, const ContentAddress & ca)
 DerivedPath read(const Store & store, Source & from, Phantom<DerivedPath> _)
 {
     auto s = readString(from);
-    return DerivedPath::parse(store, s);
+    return DerivedPath::parseLegacy(store, s);
 }
 
 void write(const Store & store, Sink & out, const DerivedPath & req)
 {
-    out << req.to_string(store);
+    out << req.to_string_legacy(store);
 }
 
 

src/libstore/sqlite.cc

@@ -239,14 +239,11 @@ SQLiteTxn::~SQLiteTxn()
     }
 }
 
-void handleSQLiteBusy(const SQLiteBusy & e)
+void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning)
 {
-    static std::atomic<time_t> lastWarned{0};
-
     time_t now = time(0);
-
+    if (now > nextWarning) {
-    if (now > lastWarned + 10) {
+        nextWarning = now + 10;
-        lastWarned = now;
         logWarning({
             .msg = hintfmt(e.what())
         });

src/libstore/sqlite.hh

@@ -139,7 +139,7 @@ protected:
 
 MakeError(SQLiteBusy, SQLiteError);
 
-void handleSQLiteBusy(const SQLiteBusy & e);
+void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning);
 
 /**
  * Convenience function for retrying a SQLite transaction when the
@@ -148,11 +148,13 @@ void handleSQLiteBusy(const SQLiteBusy & e);
 template<typename T, typename F>
 T retrySQLite(F && fun)
 {
+    time_t nextWarning = time(0) + 1;
+
     while (true) {
         try {
             return fun();
         } catch (SQLiteBusy & e) {
-            handleSQLiteBusy(e);
+            handleSQLiteBusy(e, nextWarning);
         }
     }
 }

src/libstore/store-api.hh

@@ -678,8 +678,7 @@ public:
     /**
      * @return An object to access files in the Nix store.
      */
-    virtual ref<FSAccessor> getFSAccessor()
+    virtual ref<FSAccessor> getFSAccessor() = 0;
-    { unsupported("getFSAccessor"); }
 
     /**
      * Repair the contents of the given path by redownloading it using

src/libstore/tests/derived-path.cc

@@ -51,6 +51,14 @@ TEST_F(DerivedPathTest, force_init)
 {
 }
 
+RC_GTEST_FIXTURE_PROP(
+    DerivedPathTest,
+    prop_legacy_round_rip,
+    (const DerivedPath & o))
+{
+    RC_ASSERT(o == DerivedPath::parseLegacy(*store, o.to_string_legacy(*store)));
+}
+
 RC_GTEST_FIXTURE_PROP(
     DerivedPathTest,
     prop_round_rip,

src/libutil/compression.cc

@@ -23,7 +23,7 @@ struct ChunkedCompressionSink : CompressionSink
 {
     uint8_t outbuf[32 * 1024];
 
-    void write(std::string_view data) override
+    void writeUnbuffered(std::string_view data) override
     {
         const size_t CHUNK_SIZE = sizeof(outbuf) << 2;
         while (!data.empty()) {
@@ -103,7 +103,7 @@ struct ArchiveCompressionSink : CompressionSink
             throw Error(reason, archive_error_string(this->archive));
     }
 
-    void write(std::string_view data) override
+    void writeUnbuffered(std::string_view data) override
     {
         ssize_t result = archive_write_data(archive, data.data(), data.length());
         if (result <= 0) check(result);
@@ -136,7 +136,7 @@ struct NoneSink : CompressionSink
             warn("requested compression level '%d' not supported by compression method 'none'", level);
     }
     void finish() override { flush(); }
-    void write(std::string_view data) override { nextSink(data); }
+    void writeUnbuffered(std::string_view data) override { nextSink(data); }
 };
 
 struct BrotliDecompressionSink : ChunkedCompressionSink

src/libutil/compression.hh

@@ -12,7 +12,7 @@ namespace nix {
 struct CompressionSink : BufferedSink, FinishSink
 {
     using BufferedSink::operator ();
-    using BufferedSink::write;
+    using BufferedSink::writeUnbuffered;
     using FinishSink::finish;
 };
 

src/libutil/config.cc

@@ -191,6 +191,10 @@ std::map<std::string, nlohmann::json> AbstractSetting::toJSONObject()
     std::map<std::string, nlohmann::json> obj;
     obj.emplace("description", description);
     obj.emplace("aliases", aliases);
+    if (experimentalFeature)
+        obj.emplace("experimentalFeature", *experimentalFeature);
+    else
+        obj.emplace("experimentalFeature", nullptr);
     return obj;
 }
 

src/libutil/hash.cc

@@ -1,6 +1,7 @@
 #include <iostream>
 #include <cstring>
 
+#include <openssl/crypto.h>
 #include <openssl/md5.h>
 #include <openssl/sha.h>
 
@@ -16,7 +17,6 @@
 
 namespace nix {
 
-
 static size_t regularHashSize(HashType type) {
     switch (type) {
     case htMD5: return md5HashSize;
@@ -343,7 +343,7 @@ HashSink::~HashSink()
     delete ctx;
 }
 
-void HashSink::write(std::string_view data)
+void HashSink::writeUnbuffered(std::string_view data)
 {
     bytes += data.size();
     update(ht, *ctx, data);

src/libutil/hash.hh

@@ -197,7 +197,7 @@ public:
     HashSink(HashType ht);
     HashSink(const HashSink & h);
     ~HashSink();
-    void write(std::string_view data) override;
+    void writeUnbuffered(std::string_view data) override;
     HashResult finish() override;
     HashResult currentHash();
 };

src/libutil/serialise.cc

@@ -20,7 +20,7 @@ void BufferedSink::operator () (std::string_view data)
            buffer size. */
         if (bufPos + data.size() >= bufSize) {
             flush();
-            write(data);
+            writeUnbuffered(data);
             break;
         }
         /* Otherwise, copy the bytes to the buffer.  Flush the buffer
@@ -38,7 +38,7 @@ void BufferedSink::flush()
     if (bufPos == 0) return;
     size_t n = bufPos;
     bufPos = 0; // don't trigger the assert() in ~BufferedSink()
-    write({buffer.get(), n});
+    writeUnbuffered({buffer.get(), n});
 }
 
 
@@ -48,7 +48,7 @@ FdSink::~FdSink()
 }
 
 
-void FdSink::write(std::string_view data)
+void FdSink::writeUnbuffered(std::string_view data)
 {
     written += data.size();
     try {

src/libutil/serialise.hh

@@ -53,7 +53,9 @@ struct BufferedSink : virtual Sink
 
     void flush();
 
-    virtual void write(std::string_view data) = 0;
+protected:
+
+    virtual void writeUnbuffered(std::string_view data) = 0;
 };
 
 
@@ -133,7 +135,7 @@ struct FdSink : BufferedSink
 
     ~FdSink();
 
-    void write(std::string_view data) override;
+    void writeUnbuffered(std::string_view data) override;
 
     bool good() override;
 
@@ -520,7 +522,7 @@ struct FramedSink : nix::BufferedSink
         }
     }
 
-    void write(std::string_view data) override
+    void writeUnbuffered(std::string_view data) override
     {
         /* Don't send more data if the remote has
             encountered an error. */

src/libutil/tests/config.cc

@@ -156,12 +156,54 @@ namespace nix {
     }
 
     TEST(Config, toJSONOnNonEmptyConfig) {
+        using nlohmann::literals::operator "" _json;
         Config config;
-        std::map<std::string, Config::SettingInfo> settings;
+        Setting<std::string> setting{
-        Setting<std::string> setting{&config, "", "name-of-the-setting", "description"};
+            &config,
+            "",
+            "name-of-the-setting",
+            "description",
+        };
         setting.assign("value");
 
-        ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"defaultValue":"","description":"description\n","documentDefault":true,"value":"value"}})#");
+        ASSERT_EQ(config.toJSON(),
+          R"#({
+            "name-of-the-setting": {
+              "aliases": [],
+              "defaultValue": "",
+              "description": "description\n",
+              "documentDefault": true,
+              "value": "value",
+              "experimentalFeature": null
+            }
+          })#"_json);
+    }
+
+    TEST(Config, toJSONOnNonEmptyConfigWithExperimentalSetting) {
+        using nlohmann::literals::operator "" _json;
+        Config config;
+        Setting<std::string> setting{
+            &config,
+            "",
+            "name-of-the-setting",
+            "description",
+            {},
+            true,
+            Xp::Flakes,
+        };
+        setting.assign("value");
+
+        ASSERT_EQ(config.toJSON(),
+          R"#({
+            "name-of-the-setting": {
+              "aliases": [],
+              "defaultValue": "",
+              "description": "description\n",
+              "documentDefault": true,
+              "value": "value",
+              "experimentalFeature": "flakes"
+            }
+          })#"_json);
     }
 
     TEST(Config, setSettingAlias) {

src/libutil/util.cc

@@ -47,6 +47,9 @@ extern char * * environ __attribute__((weak));
 
 namespace nix {
 
+void initLibUtil() {
+}
+
 std::optional<std::string> getEnv(const std::string & key)
 {
     char * value = getenv(key.c_str());
@@ -1744,13 +1747,39 @@ void triggerInterrupt()
 }
 
 static sigset_t savedSignalMask;
+static bool savedSignalMaskIsSet = false;
+
+void setChildSignalMask(sigset_t * sigs)
+{
+    assert(sigs); // C style function, but think of sigs as a reference
+
+#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE
+    sigemptyset(&savedSignalMask);
+    // There's no "assign" or "copy" function, so we rely on (math) idempotence
+    // of the or operator: a or a = a.
+    sigorset(&savedSignalMask, sigs, sigs);
+#else
+    // Without sigorset, our best bet is to assume that sigset_t is a type that
+    // can be assigned directly, such as is the case for a sigset_t defined as
+    // an integer type.
+    savedSignalMask = *sigs;
+#endif
+
+    savedSignalMaskIsSet = true;
+}
+
+void saveSignalMask() {
+    if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask))
+        throw SysError("querying signal mask");
+
+    savedSignalMaskIsSet = true;
+}
 
 void startSignalHandlerThread()
 {
     updateWindowSize();
 
-    if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask))
+    saveSignalMask();
-        throw SysError("querying signal mask");
 
     sigset_t set;
     sigemptyset(&set);
@@ -1767,6 +1796,20 @@ void startSignalHandlerThread()
 
 static void restoreSignals()
 {
+    // If startSignalHandlerThread wasn't called, that means we're not running
+    // in a proper libmain process, but a process that presumably manages its
+    // own signal handlers. Such a process should call either
+    //  - initNix(), to be a proper libmain process
+    //  - startSignalHandlerThread(), to resemble libmain regarding signal
+    //    handling only
+    //  - saveSignalMask(), for processes that define their own signal handling
+    //    thread
+    // TODO: Warn about this? Have a default signal mask? The latter depends on
+    //       whether we should generally inherit signal masks from the caller.
+    //       I don't know what the larger unix ecosystem expects from us here.
+    if (!savedSignalMaskIsSet)
+        return;
+
     if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr))
         throw SysError("restoring signals");
 }

src/libutil/util.hh

@@ -32,6 +32,7 @@ namespace nix {
 struct Sink;
 struct Source;
 
+void initLibUtil();
 
 /**
  * The system for which Nix is compiled.
@@ -445,6 +446,8 @@ void setStackSize(size_t stackSize);
 /**
  * Restore the original inherited Unix process context (such as signal
  * masks, stack size).
+
+ * See startSignalHandlerThread(), saveSignalMask().
  */
 void restoreProcessContext(bool restoreMounts = true);
 
@@ -814,9 +817,26 @@ class Callback;
 /**
  * Start a thread that handles various signals. Also block those signals
  * on the current thread (and thus any threads created by it).
+ * Saves the signal mask before changing the mask to block those signals.
+ * See saveSignalMask().
  */
 void startSignalHandlerThread();
 
+/**
+ * Saves the signal mask, which is the signal mask that nix will restore
+ * before creating child processes.
+ * See setChildSignalMask() to set an arbitrary signal mask instead of the
+ * current mask.
+ */
+void saveSignalMask();
+
+/**
+ * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't
+ * necessarily match the current thread's mask.
+ * See saveSignalMask() to set the saved mask to the current mask.
+ */
+void setChildSignalMask(sigset_t *sigs);
+
 struct InterruptCallback
 {
     virtual ~InterruptCallback() { };

src/nix/nix.md

@@ -48,12 +48,17 @@ manual](https://nixos.org/manual/nix/stable/).
 
 # Installables
 
+> **Warning** \
+> Installables are part of the unstable
+> [`nix-command` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-nix-command),
+> and subject to change without notice.
+
 Many `nix` subcommands operate on one or more *installables*.
 These are command line arguments that represent something that can be realised in the Nix store.
 
 The following types of installable are supported by most commands:
 
-- [Flake output attribute](#flake-output-attribute)
+- [Flake output attribute](#flake-output-attribute) (experimental)
 - [Store path](#store-path)
 - [Nix file](#nix-file), optionally qualified by an attribute path
 - [Nix expression](#nix-expression), optionally qualified by an attribute path
@@ -63,6 +68,13 @@ That is, Nix will operate on the default flake output attribute of the flake in
 
 ### Flake output attribute
 
+> **Warning** \
+> Flake output attribute installables depend on both the
+> [`flakes`](@docroot@/contributing/experimental-features.md#xp-feature-flakes)
+> and
+> [`nix-command`](@docroot@/contributing/experimental-features.md#xp-feature-nix-command)
+> experimental features, and subject to change without notice.
+
 Example: `nixpkgs#hello`
 
 These have the form *flakeref*[`#`*attrpath*], where *flakeref* is a

tests/repl.sh

@@ -79,6 +79,14 @@ testReplResponse '
 "result: ${a}"
 ' "result: 2"
 
+# check dollar escaping https://github.com/NixOS/nix/issues/4909
+# note the escaped \,
+#    \\
+# because the second argument is a regex
+testReplResponse '
+"$" + "{hi}"
+' '"\\${hi}"'
+
 testReplResponse '
 drvPath
 ' '".*-simple.drv"' \