From 9b1824ecbd222b4bdc8fa2b6f345dc55ef4872d0 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 3 Dec 2020 15:35:38 -0600 Subject: [PATCH 01/41] Add extraPlatforms for Rosetta 2 macOS macOS systems with ARM64 can utilize a translation layer at /Library/Apple/usr/libexec/oah to run x86_64 binaries. This change makes Nix recognize that and it to "extra-platforms". Note that there are two cases here since Nix could be built for either x86_64 or aarch64. In either case, we can switch to the other architecture. Unfortunately there is not a good way to prevent aarch64 binaries from being run in x86_64 contexts or vice versa - programs can always execute programs for the other architecture. --- src/libstore/globals.cc | 22 ++++++++++++++++++++++ src/libstore/globals.hh | 4 +++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index f38601d6d..59c49af8a 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -131,6 +131,28 @@ StringSet Settings::getDefaultSystemFeatures() return features; } +StringSet Settings::getDefaultExtraPlatforms() +{ + if (std::string{SYSTEM} == "x86_64-linux" && !isWSL1()) + return StringSet{"i686-linux"}; +#if __APPLE__ + // Rosetta 2 emulation layer can run x86_64 binaries on aarch64 + // machines. Note that we can’t force processes from executing + // x86_64 in aarch64 environments or vice versa since they can + // always exec with their own binary preferences. + else if (pathExists("/Library/Apple/usr/libexec/oah")) { + if (std::string{SYSTEM} == "x86_64-darwin") + return StringSet{"aarch64-darwin"}; + else if (std::string{SYSTEM} == "aarch64-darwin") + return StringSet{"x86_64-darwin"}; + else + return StringSet{}; + } +#endif + else + return StringSet{}; +} + bool Settings::isExperimentalFeatureEnabled(const std::string & name) { auto & f = experimentalFeatures.get(); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 4655ca058..8666a7d28 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -34,6 +34,8 @@ class Settings : public Config { StringSet getDefaultSystemFeatures(); + StringSet getDefaultExtraPlatforms(); + bool isWSL1(); public: @@ -545,7 +547,7 @@ public: Setting extraPlatforms{ this, - std::string{SYSTEM} == "x86_64-linux" && !isWSL1() ? StringSet{"i686-linux"} : StringSet{}, + getDefaultExtraPlatforms(), "extra-platforms", R"( Platforms other than the native one which this machine is capable of From 4b9acf4e21a834276b7d061942e7b5d3692662b6 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 3 Dec 2020 15:41:59 -0600 Subject: [PATCH 02/41] Use posix_spawn_setbinpref_np to advise which architecture to run When running universal binaries like /bin/bash, Darwin XNU will choose which architecture of the binary to use based on "binary preferences". This change sets that to the current platform for aarch64 and x86_64 builds. In addition it now uses posix_spawn instead of the usual execve. Note, that this does not prevent the other architecture from being run, just advises which to use. Unfortunately, posix_spawnattr_setbinpref_np does not appear to be inherited by child processes in x86_64 Rosetta 2 translations, meaning that this will not always work as expected. For example: { arm = derivation { name = "test"; system = "aarch64-darwin"; builder = "/bin/bash"; args = [ "-e" (builtins.toFile "test" '' set -x /usr/sbin/sysctl sysctl.proc_translated /usr/sbin/sysctl sysctl.proc_native [ "$(/usr/bin/arch)" = arm64 ] /usr/bin/touch $out '') ]; }; rosetta = derivation { name = "test"; system = "x86_64-darwin"; builder = "/bin/bash"; args = [ "-e" (builtins.toFile "test" '' set -x /usr/sbin/sysctl sysctl.proc_translated /usr/sbin/sysctl sysctl.proc_native [ "$(/usr/bin/arch)" = i386 ] echo It works! /usr/bin/touch $out '') ]; }; } `arm' fails on x86_64-compiled Nix, but `arm' and `rosetta' succeed on aarch64-compiled Nix. I suspect there is a way to fix this since: $ /usr/bin/arch -arch x86_64 /bin/bash \ -c '/usr/bin/arch -arch arm64e /bin/bash -c /usr/bin/arch' arm64 seems to work correctly. We may need to wait for Apple to update system_cmds in opensource.apple.com to find out how though. --- src/libstore/build/derivation-goal.cc | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 1db85bd37..f370fd82d 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -50,6 +50,10 @@ #define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) #endif +#if __APPLE__ +#include +#endif + #include #include @@ -2844,7 +2848,27 @@ void DerivationGoal::runChild() } } +#if __APPLE__ + posix_spawnattr_t attrp; + + if (posix_spawnattr_init(&attrp)) + throw SysError("failed to initialize builder"); + + if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC)) + throw SysError("failed to initialize builder"); + + if (drv->platform == "aarch64-darwin") { + cpu_type_t cpu = CPU_TYPE_ARM64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } else if (drv->platform == "x86_64-darwin") { + cpu_type_t cpu = CPU_TYPE_X86_64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } + + posix_spawn(NULL, builder, NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); +#else execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); +#endif throw SysError("executing '%1%'", drv->builder); From 3c9b7029ba88e8b831f2054c085ab1fc55c31673 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 4 Dec 2020 13:26:53 -0600 Subject: [PATCH 03/41] Use com.apple.oahd.plist for rosetta 2 detection --- src/libstore/globals.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 59c49af8a..ad66ef8a8 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -140,7 +140,7 @@ StringSet Settings::getDefaultExtraPlatforms() // machines. Note that we can’t force processes from executing // x86_64 in aarch64 environments or vice versa since they can // always exec with their own binary preferences. - else if (pathExists("/Library/Apple/usr/libexec/oah")) { + else if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) { if (std::string{SYSTEM} == "x86_64-darwin") return StringSet{"aarch64-darwin"}; else if (std::string{SYSTEM} == "aarch64-darwin") From d4870462f8f539adeaa6dca476aff6f1f31e1981 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 8 Dec 2020 14:16:06 -0600 Subject: [PATCH 04/41] Cast variants fully for libc++10 libc++10 seems to be stricter on what it allows in variant conversion. I'm not sure what the rules are here, but this is the minimal change needed to get through the compilation errors. --- src/libexpr/eval-cache.cc | 2 +- src/libexpr/primops/fetchTree.cc | 2 +- src/libfetchers/attrs.cc | 2 +- src/libfetchers/github.cc | 4 ++-- src/libfetchers/mercurial.cc | 2 +- src/libfetchers/tarball.cc | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 7b025be23..539ba71f3 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -394,7 +394,7 @@ Value & AttrCursor::forceValue() cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), string_t{v.string.s, {}}}; else if (v.type == tPath) - cachedValue = {root->db->setString(getKey(), v.path), v.path}; + cachedValue = {root->db->setString(getKey(), v.path), string_t{v.path, {}}}; else if (v.type == tBool) cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean}; else if (v.type == tAttrs) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index d094edf92..1360ade39 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -104,7 +104,7 @@ static void fetchTree( else if (attr.value->type == tBool) attrs.emplace(attr.name, Explicit{attr.value->boolean}); else if (attr.value->type == tInt) - attrs.emplace(attr.name, attr.value->integer); + attrs.emplace(attr.name, uint64_t(attr.value->integer)); else throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected", attr.name, showType(*attr.value)); diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index 720b19fcd..17fc4041f 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -11,7 +11,7 @@ Attrs jsonToAttrs(const nlohmann::json & json) for (auto & i : json.items()) { if (i.value().is_number()) - attrs.emplace(i.key(), i.value().get()); + attrs.emplace(i.key(), i.value().get()); else if (i.value().is_string()) attrs.emplace(i.key(), i.value().get()); else if (i.value().is_boolean()) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 92ff224f7..db1ced5d6 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -195,14 +195,14 @@ struct GitArchiveInputScheme : InputScheme auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers); - input.attrs.insert_or_assign("lastModified", lastModified); + input.attrs.insert_or_assign("lastModified", uint64_t(lastModified)); getCache()->add( store, immutableAttrs, { {"rev", rev->gitRev()}, - {"lastModified", lastModified} + {"lastModified", uint64_t(lastModified)} }, tree.storePath, true); diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 07a51059d..0eb401e10 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -301,7 +301,7 @@ struct MercurialInputScheme : InputScheme Attrs infoAttrs({ {"rev", input.getRev()->gitRev()}, - {"revCount", (int64_t) revCount}, + {"revCount", (uint64_t) revCount}, }); if (!_input.getRev()) diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 8c0f20475..56c014a8c 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -152,7 +152,7 @@ std::pair downloadTarball( } Attrs infoAttrs({ - {"lastModified", lastModified}, + {"lastModified", uint64_t(lastModified)}, {"etag", res.etag}, }); From 9d3aad7b92762973081738a8bc6a562fda45c341 Mon Sep 17 00:00:00 2001 From: Sevan Janiyan Date: Fri, 25 Dec 2020 01:43:22 +0000 Subject: [PATCH 05/41] Update URL where bzip2 can be obtained --- doc/manual/src/installation/prerequisites-source.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/src/installation/prerequisites-source.md index 69b7c5a5e..6825af707 100644 --- a/doc/manual/src/installation/prerequisites-source.md +++ b/doc/manual/src/installation/prerequisites-source.md @@ -30,7 +30,7 @@ have bzip2 installed, including development headers and libraries. If your distribution does not provide these, you can obtain bzip2 from - . + . - `liblzma`, which is provided by XZ Utils. If your distribution does not provide this, you can get it from . From f1e9bda9d1ece9be8c78f5c5345c3adb299bc4aa Mon Sep 17 00:00:00 2001 From: Sevan Janiyan Date: Fri, 25 Dec 2020 01:48:21 +0000 Subject: [PATCH 06/41] Update URL where bzip2 can be obtained --- configure.ac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index daf378997..c1bfc9b53 100644 --- a/configure.ac +++ b/configure.ac @@ -174,9 +174,9 @@ PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"] # Look for libbz2, a required dependency. AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true], - [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])]) + [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://sourceware.org/bzip2/.])]) AC_CHECK_HEADERS([bzlib.h], [true], - [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])]) + [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://sourceware.org/bzip2/.])]) # Checks for libarchive PKG_CHECK_MODULES([LIBARCHIVE], [libarchive >= 3.1.2], [CXXFLAGS="$LIBARCHIVE_CFLAGS $CXXFLAGS"]) # Workaround until https://github.com/libarchive/libarchive/issues/1446 is fixed From 6262a703636000e525d5c1b877ac28d604a493f0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 28 Dec 2020 17:21:19 +0100 Subject: [PATCH 07/41] scanForReferences: Remove misleading comment References have always been determined only by the hash part, not the name or the store prefix. Fixes #4396. --- src/libstore/references.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/libstore/references.cc b/src/libstore/references.cc index eb117b5ba..39c4970c6 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -88,9 +88,6 @@ PathSet scanForReferences(Sink & toTee, TeeSink sink { refsSink, toTee }; std::map backMap; - /* For efficiency (and a higher hit rate), just search for the - hash part of the file name. (This assumes that all references - have the form `HASH-bla'). */ for (auto & i : refs) { auto baseName = std::string(baseNameOf(i)); string::size_type pos = baseName.find('-'); From 093de16223b8b93d803e4cd1cc1d3945cb3dfeb1 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Mon, 28 Dec 2020 09:30:14 -0800 Subject: [PATCH 08/41] README: fix link to hacking guide --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 11fe5f932..4686010ef 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Information on additional installation methods is available on the [Nix download ## Building And Developing -See our [Hacking guide](https://hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual/hacking.html) in our manual for instruction on how to +See our [Hacking guide](https://hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual/contributing/hacking.html) in our manual for instruction on how to build nix from source with nix-build or how to get a development environment. ## Additional Resources From 64904b9d5d32c4201aaf462ae82b736f33785793 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 28 Dec 2020 19:40:04 -0600 Subject: [PATCH 09/41] Fixup --- src/libexpr/eval-cache.cc | 4 ++-- src/libexpr/primops/fetchTree.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 0f84944cd..98d91c905 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -393,9 +393,9 @@ Value & AttrCursor::forceValue() if (v.type() == nString) cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), string_t{v.string.s, {}}}; - else if (v.type == nPath) + else if (v.type() == nPath) cachedValue = {root->db->setString(getKey(), v.path), string_t{v.path, {}}}; - else if (v.type == nBool) + else if (v.type() == nBool) cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean}; else if (v.type() == nAttrs) ; // FIXME: do something? diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index e64e3fbb8..ab80be2d3 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -103,7 +103,7 @@ static void fetchTree( addURI(state, attrs, attr.name, attr.value->string.s); else if (attr.value->type() == nBool) attrs.emplace(attr.name, Explicit{attr.value->boolean}); - else if (attr.value->type == nInt) + else if (attr.value->type() == nInt) attrs.emplace(attr.name, uint64_t(attr.value->integer)); else throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected", From d27eb0ef573b4739967119448779da4a8b2a2cbf Mon Sep 17 00:00:00 2001 From: David McFarland Date: Wed, 30 Dec 2020 16:20:03 -0400 Subject: [PATCH 10/41] Fix insufficent attribute capacity in user profile --- src/nix-env/user-env.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 87387e794..168ac492b 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -53,10 +53,12 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, output paths, and optionally the derivation path, as well as the meta attributes. */ Path drvPath = keepDerivations ? i.queryDrvPath() : ""; + DrvInfo::Outputs outputs = i.queryOutputs(true); + StringSet metaNames = i.queryMetaNames(); Value & v(*state.allocValue()); manifest.listElems()[n++] = &v; - state.mkAttrs(v, 16); + state.mkAttrs(v, 7 + outputs.size()); mkString(*state.allocAttr(v, state.sType), "derivation"); mkString(*state.allocAttr(v, state.sName), i.queryName()); @@ -68,7 +70,6 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, mkString(*state.allocAttr(v, state.sDrvPath), i.queryDrvPath()); // Copy each output meant for installation. - DrvInfo::Outputs outputs = i.queryOutputs(true); Value & vOutputs = *state.allocAttr(v, state.sOutputs); state.mkList(vOutputs, outputs.size()); unsigned int m = 0; @@ -88,8 +89,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, // Copy the meta attributes. Value & vMeta = *state.allocAttr(v, state.sMeta); - state.mkAttrs(vMeta, 16); - StringSet metaNames = i.queryMetaNames(); + state.mkAttrs(vMeta, metaNames.size()); for (auto & j : metaNames) { Value * v = i.queryMeta(j); if (!v) continue; From e069ddf3258b7eab2074639cade03ba03b0a03a4 Mon Sep 17 00:00:00 2001 From: Sam Lidder Date: Thu, 31 Dec 2020 20:17:37 -0500 Subject: [PATCH 11/41] Fix `configure` error in introduction doc --- doc/manual/src/introduction.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/introduction.md b/doc/manual/src/introduction.md index f01fe7b38..d68445c95 100644 --- a/doc/manual/src/introduction.md +++ b/doc/manual/src/introduction.md @@ -165,10 +165,10 @@ You’re then dropped into a shell where you can edit, build and test the package: ```console -[nix-shell]$ tar xf $src +[nix-shell]$ unpackPhase [nix-shell]$ cd pan-* -[nix-shell]$ ./configure -[nix-shell]$ make +[nix-shell]$ configurePhase +[nix-shell]$ buildPhase [nix-shell]$ ./pan/gui/pan ``` From 988dd0a65f562741708f6a7a7a44e333d6a5b205 Mon Sep 17 00:00:00 2001 From: Danila Fedorin Date: Tue, 5 Jan 2021 02:06:25 +0000 Subject: [PATCH 12/41] Fix conversion from JSON to fetch attributes It appears as through the fetch attribute, which is simply a variant with 3 elements, implicitly converts boolean arguments to integers. One must use Explicit to correctly populate it with a boolean. This was missing from the implementation, and resulted in clearly boolean JSON fields being treated as numbers. --- src/libfetchers/attrs.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index 17fc4041f..a565d19d4 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -15,7 +15,7 @@ Attrs jsonToAttrs(const nlohmann::json & json) else if (i.value().is_string()) attrs.emplace(i.key(), i.value().get()); else if (i.value().is_boolean()) - attrs.emplace(i.key(), i.value().get()); + attrs.emplace(i.key(), Explicit { i.value().get() }); else throw Error("unsupported input attribute type in lock file"); } From 8af4f886e212346afdd1d40789f96f1321da96c5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 Jan 2021 11:47:29 +0100 Subject: [PATCH 13/41] Fix deadlock in LocalStore::addSignatures() Fixes #4367. --- src/libstore/local-store.cc | 95 +++++++++++++++++++------------------ src/libstore/local-store.hh | 2 + 2 files changed, 52 insertions(+), 45 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c52d4b62a..702e7b136 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -736,57 +736,62 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, Callback> callback) noexcept { try { - callback(retrySQLite>([&]() { + callback(retrySQLite>([&]() { auto state(_state.lock()); - - /* Get the path info. */ - auto useQueryPathInfo(state->stmts->QueryPathInfo.use()(printStorePath(path))); - - if (!useQueryPathInfo.next()) - return std::shared_ptr(); - - auto id = useQueryPathInfo.getInt(0); - - auto narHash = Hash::dummy; - try { - narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1)); - } catch (BadHash & e) { - throw Error("invalid-path entry for '%s': %s", printStorePath(path), e.what()); - } - - auto info = std::make_shared(path, narHash); - - info->id = id; - - info->registrationTime = useQueryPathInfo.getInt(2); - - auto s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 3); - if (s) info->deriver = parseStorePath(s); - - /* Note that narSize = NULL yields 0. */ - info->narSize = useQueryPathInfo.getInt(4); - - info->ultimate = useQueryPathInfo.getInt(5) == 1; - - s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 6); - if (s) info->sigs = tokenizeString(s, " "); - - s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 7); - if (s) info->ca = parseContentAddressOpt(s); - - /* Get the references. */ - auto useQueryReferences(state->stmts->QueryReferences.use()(info->id)); - - while (useQueryReferences.next()) - info->references.insert(parseStorePath(useQueryReferences.getStr(0))); - - return info; + return queryPathInfoInternal(*state, path); })); } catch (...) { callback.rethrow(); } } +std::shared_ptr LocalStore::queryPathInfoInternal(State & state, const StorePath & path) +{ + /* Get the path info. */ + auto useQueryPathInfo(state.stmts->QueryPathInfo.use()(printStorePath(path))); + + if (!useQueryPathInfo.next()) + return std::shared_ptr(); + + auto id = useQueryPathInfo.getInt(0); + + auto narHash = Hash::dummy; + try { + narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1)); + } catch (BadHash & e) { + throw Error("invalid-path entry for '%s': %s", printStorePath(path), e.what()); + } + + auto info = std::make_shared(path, narHash); + + info->id = id; + + info->registrationTime = useQueryPathInfo.getInt(2); + + auto s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 3); + if (s) info->deriver = parseStorePath(s); + + /* Note that narSize = NULL yields 0. */ + info->narSize = useQueryPathInfo.getInt(4); + + info->ultimate = useQueryPathInfo.getInt(5) == 1; + + s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 6); + if (s) info->sigs = tokenizeString(s, " "); + + s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7); + if (s) info->ca = parseContentAddressOpt(s); + + /* Get the references. */ + auto useQueryReferences(state.stmts->QueryReferences.use()(info->id)); + + while (useQueryReferences.next()) + info->references.insert(parseStorePath(useQueryReferences.getStr(0))); + + return info; +} + + /* Update path info in the database. */ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { @@ -1608,7 +1613,7 @@ void LocalStore::addSignatures(const StorePath & storePath, const StringSet & si SQLiteTxn txn(state->db); - auto info = std::const_pointer_cast(std::shared_ptr(queryPathInfo(storePath))); + auto info = std::const_pointer_cast(queryPathInfoInternal(*state, storePath)); info->sigs.insert(sigs.begin(), sigs.end()); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index ae9497b2e..6d29c5960 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -235,6 +235,8 @@ private: void verifyPath(const Path & path, const StringSet & store, PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); + std::shared_ptr queryPathInfoInternal(State & state, const StorePath & path); + void updatePathInfo(State & state, const ValidPathInfo & info); void upgradeStore6(); From 146af4ee9bb03968a7322a1ac70dc60c8d5a35e2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Jan 2021 16:43:09 +0100 Subject: [PATCH 14/41] Move sodium_init() call --- src/libmain/shared.cc | 9 +++++++++ src/nix-store/nix-store.cc | 3 --- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 2247aeca4..e9f067e35 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -18,6 +18,10 @@ #include +#if HAVE_SODIUM +#include +#endif + namespace nix { @@ -126,6 +130,11 @@ void initNix() CRYPTO_set_locking_callback(opensslLockCallback); #endif +#if HAVE_SODIUM + if (sodium_init() == -1) + throw Error("could not initialise libsodium"); +#endif + loadConfFile(); startSignalHandlerThread(); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 54394e921..e1ccece99 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -981,9 +981,6 @@ static void opGenerateBinaryCacheKey(Strings opFlags, Strings opArgs) string publicKeyFile = *i++; #if HAVE_SODIUM - if (sodium_init() == -1) - throw Error("could not initialise libsodium"); - unsigned char pk[crypto_sign_PUBLICKEYBYTES]; unsigned char sk[crypto_sign_SECRETKEYBYTES]; if (crypto_sign_keypair(pk, sk) != 0) From 555152ffe8494190ca42dd481991c9b54759f686 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Jan 2021 17:04:46 +0100 Subject: [PATCH 15/41] crypto.cc: API cleanup and add generate() / to_string() methods --- src/libstore/crypto.cc | 33 ++++++++++++++++++++++++++------- src/libstore/crypto.hh | 24 ++++++++++++++++-------- src/nix-store/nix-store.cc | 17 +++-------------- 3 files changed, 45 insertions(+), 29 deletions(-) diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc index 9ec8abd22..135ced277 100644 --- a/src/libstore/crypto.cc +++ b/src/libstore/crypto.cc @@ -8,15 +8,15 @@ namespace nix { -static std::pair split(const string & s) +static std::pair split(std::string_view s) { size_t colon = s.find(':'); if (colon == std::string::npos || colon == 0) return {"", ""}; - return {std::string(s, 0, colon), std::string(s, colon + 1)}; + return {s.substr(0, colon), s.substr(colon + 1)}; } -Key::Key(const string & s) +Key::Key(std::string_view s) { auto ss = split(s); @@ -29,7 +29,12 @@ Key::Key(const string & s) key = base64Decode(key); } -SecretKey::SecretKey(const string & s) +std::string Key::to_string() const +{ + return name + ":" + base64Encode(key); +} + +SecretKey::SecretKey(std::string_view s) : Key(s) { #if HAVE_SODIUM @@ -45,7 +50,7 @@ SecretKey::SecretKey(const string & s) } #endif -std::string SecretKey::signDetached(const std::string & data) const +std::string SecretKey::signDetached(std::string_view data) const { #if HAVE_SODIUM unsigned char sig[crypto_sign_BYTES]; @@ -69,7 +74,21 @@ PublicKey SecretKey::toPublicKey() const #endif } -PublicKey::PublicKey(const string & s) +SecretKey SecretKey::generate(std::string_view name) +{ +#if HAVE_SODIUM + unsigned char pk[crypto_sign_PUBLICKEYBYTES]; + unsigned char sk[crypto_sign_SECRETKEYBYTES]; + if (crypto_sign_keypair(pk, sk) != 0) + throw Error("key generation failed"); + + return SecretKey(name, std::string((char *) sk, crypto_sign_SECRETKEYBYTES)); +#else + noSodium(); +#endif +} + +PublicKey::PublicKey(std::string_view s) : Key(s) { #if HAVE_SODIUM @@ -84,7 +103,7 @@ bool verifyDetached(const std::string & data, const std::string & sig, #if HAVE_SODIUM auto ss = split(sig); - auto key = publicKeys.find(ss.first); + auto key = publicKeys.find(std::string(ss.first)); if (key == publicKeys.end()) return false; auto sig2 = base64Decode(ss.second); diff --git a/src/libstore/crypto.hh b/src/libstore/crypto.hh index 9110af3aa..03f85c103 100644 --- a/src/libstore/crypto.hh +++ b/src/libstore/crypto.hh @@ -13,32 +13,40 @@ struct Key /* Construct Key from a string in the format ‘:’. */ - Key(const std::string & s); + Key(std::string_view s); + + std::string to_string() const; protected: - Key(const std::string & name, const std::string & key) - : name(name), key(key) { } + Key(std::string_view name, std::string && key) + : name(name), key(std::move(key)) { } }; struct PublicKey; struct SecretKey : Key { - SecretKey(const std::string & s); + SecretKey(std::string_view s); /* Return a detached signature of the given string. */ - std::string signDetached(const std::string & s) const; + std::string signDetached(std::string_view s) const; PublicKey toPublicKey() const; + + static SecretKey generate(std::string_view name); + +private: + SecretKey(std::string_view name, std::string && key) + : Key(name, std::move(key)) { } }; struct PublicKey : Key { - PublicKey(const std::string & data); + PublicKey(std::string_view data); private: - PublicKey(const std::string & name, const std::string & key) - : Key(name, key) { } + PublicKey(std::string_view name, std::string && key) + : Key(name, std::move(key)) { } friend struct SecretKey; }; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index e1ccece99..e43788bc3 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -19,10 +19,6 @@ #include #include -#if HAVE_SODIUM -#include -#endif - namespace nix_store { @@ -980,18 +976,11 @@ static void opGenerateBinaryCacheKey(Strings opFlags, Strings opArgs) string secretKeyFile = *i++; string publicKeyFile = *i++; -#if HAVE_SODIUM - unsigned char pk[crypto_sign_PUBLICKEYBYTES]; - unsigned char sk[crypto_sign_SECRETKEYBYTES]; - if (crypto_sign_keypair(pk, sk) != 0) - throw Error("key generation failed"); + auto secretKey = SecretKey::generate(keyName); - writeFile(publicKeyFile, keyName + ":" + base64Encode(string((char *) pk, crypto_sign_PUBLICKEYBYTES))); + writeFile(publicKeyFile, secretKey.toPublicKey().to_string()); umask(0077); - writeFile(secretKeyFile, keyName + ":" + base64Encode(string((char *) sk, crypto_sign_SECRETKEYBYTES))); -#else - throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); -#endif + writeFile(secretKeyFile, secretKey.to_string()); } From 9374c2baeabe45a22e4b8746dc97f5ce4f030184 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Jan 2021 17:41:16 +0100 Subject: [PATCH 16/41] Add commands for generating secret/public keys --- src/nix/hash.cc | 5 -- src/nix/key-convert-secret-to-public.md | 19 ++++++ src/nix/key-generate-secret.md | 48 ++++++++++++++ src/nix/sigs.cc | 87 +++++++++++++++++++++++++ tests/binary-cache.sh | 12 ++-- 5 files changed, 160 insertions(+), 11 deletions(-) create mode 100644 src/nix/key-convert-secret-to-public.md create mode 100644 src/nix/key-generate-secret.md diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 101b67e6a..6fd791f41 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -132,11 +132,6 @@ struct CmdHash : NixMultiCommand command->second->prepare(); command->second->run(); } - - void printHelp(const string & programName, std::ostream & out) override - { - MultiCommand::printHelp(programName, out); - } }; static auto rCmdHash = registerCommand("hash"); diff --git a/src/nix/key-convert-secret-to-public.md b/src/nix/key-convert-secret-to-public.md new file mode 100644 index 000000000..3adc18502 --- /dev/null +++ b/src/nix/key-convert-secret-to-public.md @@ -0,0 +1,19 @@ +R""( + +# Examples + +* Convert a secret key to a public key: + + ```console + # echo cache.example.org-0:E7lAO+MsPwTFfPXsdPtW8GKui/5ho4KQHVcAGnX+Tti1V4dUxoVoqLyWJ4YESuZJwQ67GVIksDt47og+tPVUZw== \ + | nix key convert-secret-to-public + cache.example.org-0:tVeHVMaFaKi8lieGBErmScEOuxlSJLA7eO6IPrT1VGc= + ``` + +# Description + +This command reads a Ed25519 secret key from standard input, and +writes the corresponding public key to standard output. For more +details, see [nix key generate-secret](./nix3-key-generate-secret.md). + +)"" diff --git a/src/nix/key-generate-secret.md b/src/nix/key-generate-secret.md new file mode 100644 index 000000000..6ff1e1c9b --- /dev/null +++ b/src/nix/key-generate-secret.md @@ -0,0 +1,48 @@ +R""( + +# Examples + +* Generate a new secret key: + + ```console + # nix key generate-secret --key-name cache.example.org-1 > ./secret-key + ``` + + We can then use this key to sign the closure of the Hello package: + + ```console + # nix build nixpkgs#hello + # nix store sign-paths --key-file ./secret-key --recursive ./result + ``` + + Finally, we can verify the store paths using the corresponding + public key: + + ``` + # nix store verify --trusted-public-keys $(nix key convert-secret-to-public < ./secret-key) ./result + ``` + +# Description + +This command generates a new Ed25519 secret key for signing store +paths and prints it on standard output. Use `nix key +convert-secret-to-public` to get the corresponding public key for +verifying signed store paths. + +The mandatory argument `--key-name` specifies a key name (such as +`cache.example.org-1). It is used to look up keys on the client when +it verifies signatures. It can be anything, but it’s suggested to use +the host name of your cache (e.g. `cache.example.org`) with a suffix +denoting the number of the key (to be incremented every time you need +to revoke a key). + +# Format + +Both secret and public keys are represented as the key name followed +by a base-64 encoding of the Ed25519 key data, e.g. + +``` +cache.example.org-0:E7lAO+MsPwTFfPXsdPtW8GKui/5ho4KQHVcAGnX+Tti1V4dUxoVoqLyWJ4YESuZJwQ67GVIksDt47og+tPVUZw== +``` + +)"" diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 37b8a6712..b2e598ad5 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -141,3 +141,90 @@ struct CmdSignPaths : StorePathsCommand }; static auto rCmdSignPaths = registerCommand2({"store", "sign-paths"}); + +#if HAVE_SODIUM +struct CmdKeyGenerateSecret : Command +{ + std::optional keyName; + + CmdKeyGenerateSecret() + { + addFlag({ + .longName = "key-name", + .description = "identifier of the key (e.g. `cache.example.org-1`)", + .labels = {"name"}, + .handler = {&keyName}, + }); + } + + std::string description() override + { + return "generate a secret key for signing store paths"; + } + + std::string doc() override + { + return + #include "key-generate-secret.md" + ; + } + + void run() override + { + if (!keyName) + throw UsageError("required argument '--key-name' is missing"); + + std::cout << SecretKey::generate(*keyName).to_string(); + } +}; + +struct CmdKeyConvertSecretToPublic : Command +{ + std::string description() override + { + return "generate a public key for verifying store paths from a secret key read from standard input"; + } + + std::string doc() override + { + return + #include "key-convert-secret-to-public.md" + ; + } + + void run() override + { + SecretKey secretKey(drainFD(STDIN_FILENO)); + std::cout << secretKey.toPublicKey().to_string(); + } +}; + +struct CmdKey : NixMultiCommand +{ + CmdKey() + : MultiCommand({ + {"generate-secret", []() { return make_ref(); }}, + {"convert-secret-to-public", []() { return make_ref(); }}, + }) + { + } + + std::string description() override + { + return "generate and convert Nix signing keys"; + } + + Category category() override { return catUtility; } + + void run() override + { + if (!command) + throw UsageError("'nix flake' requires a sub-command."); + settings.requireExperimentalFeature("flakes"); + command->second->prepare(); + command->second->run(); + } +}; + +static auto rCmdKey = registerCommand("key"); +#endif diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index 92ed36225..1a06404ed 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -131,14 +131,14 @@ if [ -n "$HAVE_SODIUM" ]; then clearCache clearCacheCache -declare -a res=($(nix-store --generate-binary-cache-key test.nixos.org-1 $TEST_ROOT/sk1 $TEST_ROOT/pk1 )) -publicKey="$(cat $TEST_ROOT/pk1)" +nix key generate-secret --key-name test.nixos.org-1 > $TEST_ROOT/sk1 +publicKey=$(nix key convert-secret-to-public < $TEST_ROOT/sk1) -res=($(nix-store --generate-binary-cache-key test.nixos.org-1 $TEST_ROOT/sk2 $TEST_ROOT/pk2)) -badKey="$(cat $TEST_ROOT/pk2)" +nix key generate-secret --key-name test.nixos.org-1 > $TEST_ROOT/sk2 +badKey=$(nix key convert-secret-to-public < $TEST_ROOT/sk2) -res=($(nix-store --generate-binary-cache-key foo.nixos.org-1 $TEST_ROOT/sk3 $TEST_ROOT/pk3)) -otherKey="$(cat $TEST_ROOT/pk3)" +nix key generate-secret --key-name foo.nixos.org-1 > $TEST_ROOT/sk3 +otherKey=$(nix key convert-secret-to-public < $TEST_ROOT/sk3) _NIX_FORCE_HTTP= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath From 0df69d96e02ce4c9e17bd33333c5d78313341dd3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Jan 2021 17:56:53 +0100 Subject: [PATCH 17/41] Make sodium a required dependency --- Makefile.config.in | 1 - configure.ac | 6 +----- perl/Makefile.config.in | 1 - perl/configure.ac | 6 +----- perl/lib/Nix/Store.xs | 10 ---------- src/libmain/shared.cc | 6 ------ src/libstore/crypto.cc | 29 ----------------------------- src/nix/sigs.cc | 2 -- tests/binary-cache.sh | 4 ---- tests/common.sh.in | 1 - 10 files changed, 2 insertions(+), 64 deletions(-) diff --git a/Makefile.config.in b/Makefile.config.in index 3845b3be0..d1e59e4e7 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -10,7 +10,6 @@ EDITLINE_LIBS = @EDITLINE_LIBS@ ENABLE_S3 = @ENABLE_S3@ GTEST_LIBS = @GTEST_LIBS@ HAVE_SECCOMP = @HAVE_SECCOMP@ -HAVE_SODIUM = @HAVE_SODIUM@ LDFLAGS = @LDFLAGS@ LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@ LIBBROTLI_LIBS = @LIBBROTLI_LIBS@ diff --git a/configure.ac b/configure.ac index c1bfc9b53..2047ed8d2 100644 --- a/configure.ac +++ b/configure.ac @@ -203,11 +203,7 @@ PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLA ]) # Look for libsodium, an optional dependency. -PKG_CHECK_MODULES([SODIUM], [libsodium], - [AC_DEFINE([HAVE_SODIUM], [1], [Whether to use libsodium for cryptography.]) - CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS" - have_sodium=1], [have_sodium=]) -AC_SUBST(HAVE_SODIUM, [$have_sodium]) +PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) # Look for liblzma, a required dependency. PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"]) diff --git a/perl/Makefile.config.in b/perl/Makefile.config.in index c87d4817e..eccfbd9f6 100644 --- a/perl/Makefile.config.in +++ b/perl/Makefile.config.in @@ -2,7 +2,6 @@ CC = @CC@ CFLAGS = @CFLAGS@ CXX = @CXX@ CXXFLAGS = @CXXFLAGS@ -HAVE_SODIUM = @HAVE_SODIUM@ PACKAGE_NAME = @PACKAGE_NAME@ PACKAGE_VERSION = @PACKAGE_VERSION@ SODIUM_LIBS = @SODIUM_LIBS@ diff --git a/perl/configure.ac b/perl/configure.ac index 255744afd..85183c005 100644 --- a/perl/configure.ac +++ b/perl/configure.ac @@ -40,11 +40,7 @@ AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname]) AC_MSG_RESULT($perllibdir) # Look for libsodium, an optional dependency. -PKG_CHECK_MODULES([SODIUM], [libsodium], - [AC_DEFINE([HAVE_SODIUM], [1], [Whether to use libsodium for cryptography.]) - CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS" - have_sodium=1], [have_sodium=]) -AC_SUBST(HAVE_SODIUM, [$have_sodium]) +PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) # Check for the required Perl dependencies (DBI and DBD::SQLite). perlFlags="-I$perllibdir" diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 9e3b7d389..ad9042a2a 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -14,9 +14,7 @@ #include "util.hh" #include "crypto.hh" -#if HAVE_SODIUM #include -#endif using namespace nix; @@ -239,12 +237,8 @@ SV * convertHash(char * algo, char * s, int toBase32) SV * signString(char * secretKey_, char * msg) PPCODE: try { -#if HAVE_SODIUM auto sig = SecretKey(secretKey_).signDetached(msg); XPUSHs(sv_2mortal(newSVpv(sig.c_str(), sig.size()))); -#else - throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); -#endif } catch (Error & e) { croak("%s", e.what()); } @@ -253,7 +247,6 @@ SV * signString(char * secretKey_, char * msg) int checkSignature(SV * publicKey_, SV * sig_, char * msg) CODE: try { -#if HAVE_SODIUM STRLEN publicKeyLen; unsigned char * publicKey = (unsigned char *) SvPV(publicKey_, publicKeyLen); if (publicKeyLen != crypto_sign_PUBLICKEYBYTES) @@ -265,9 +258,6 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg) throw Error("signature is not valid"); RETVAL = crypto_sign_verify_detached(sig, (unsigned char *) msg, strlen(msg), publicKey) == 0; -#else - throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); -#endif } catch (Error & e) { croak("%s", e.what()); } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index e9f067e35..6751a3744 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -18,9 +18,7 @@ #include -#if HAVE_SODIUM #include -#endif namespace nix { @@ -130,10 +128,8 @@ void initNix() CRYPTO_set_locking_callback(opensslLockCallback); #endif -#if HAVE_SODIUM if (sodium_init() == -1) throw Error("could not initialise libsodium"); -#endif loadConfFile(); @@ -283,9 +279,7 @@ void printVersion(const string & programName) #if HAVE_BOEHMGC cfg.push_back("gc"); #endif -#if HAVE_SODIUM cfg.push_back("signed-caches"); -#endif std::cout << "System type: " << settings.thisSystem << "\n"; std::cout << "Additional system types: " << concatStringsSep(", ", settings.extraPlatforms.get()) << "\n"; std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n"; diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc index 135ced277..1027469c9 100644 --- a/src/libstore/crypto.cc +++ b/src/libstore/crypto.cc @@ -2,9 +2,7 @@ #include "util.hh" #include "globals.hh" -#if HAVE_SODIUM #include -#endif namespace nix { @@ -37,70 +35,46 @@ std::string Key::to_string() const SecretKey::SecretKey(std::string_view s) : Key(s) { -#if HAVE_SODIUM if (key.size() != crypto_sign_SECRETKEYBYTES) throw Error("secret key is not valid"); -#endif } -#if !HAVE_SODIUM -[[noreturn]] static void noSodium() -{ - throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); -} -#endif - std::string SecretKey::signDetached(std::string_view data) const { -#if HAVE_SODIUM unsigned char sig[crypto_sign_BYTES]; unsigned long long sigLen; crypto_sign_detached(sig, &sigLen, (unsigned char *) data.data(), data.size(), (unsigned char *) key.data()); return name + ":" + base64Encode(std::string((char *) sig, sigLen)); -#else - noSodium(); -#endif } PublicKey SecretKey::toPublicKey() const { -#if HAVE_SODIUM unsigned char pk[crypto_sign_PUBLICKEYBYTES]; crypto_sign_ed25519_sk_to_pk(pk, (unsigned char *) key.data()); return PublicKey(name, std::string((char *) pk, crypto_sign_PUBLICKEYBYTES)); -#else - noSodium(); -#endif } SecretKey SecretKey::generate(std::string_view name) { -#if HAVE_SODIUM unsigned char pk[crypto_sign_PUBLICKEYBYTES]; unsigned char sk[crypto_sign_SECRETKEYBYTES]; if (crypto_sign_keypair(pk, sk) != 0) throw Error("key generation failed"); return SecretKey(name, std::string((char *) sk, crypto_sign_SECRETKEYBYTES)); -#else - noSodium(); -#endif } PublicKey::PublicKey(std::string_view s) : Key(s) { -#if HAVE_SODIUM if (key.size() != crypto_sign_PUBLICKEYBYTES) throw Error("public key is not valid"); -#endif } bool verifyDetached(const std::string & data, const std::string & sig, const PublicKeys & publicKeys) { -#if HAVE_SODIUM auto ss = split(sig); auto key = publicKeys.find(std::string(ss.first)); @@ -113,9 +87,6 @@ bool verifyDetached(const std::string & data, const std::string & sig, return crypto_sign_verify_detached((unsigned char *) sig2.data(), (unsigned char *) data.data(), data.size(), (unsigned char *) key->second.key.data()) == 0; -#else - noSodium(); -#endif } PublicKeys getDefaultPublicKeys() diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index b2e598ad5..14e2c9761 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -142,7 +142,6 @@ struct CmdSignPaths : StorePathsCommand static auto rCmdSignPaths = registerCommand2({"store", "sign-paths"}); -#if HAVE_SODIUM struct CmdKeyGenerateSecret : Command { std::optional keyName; @@ -227,4 +226,3 @@ struct CmdKey : NixMultiCommand }; static auto rCmdKey = registerCommand("key"); -#endif diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index 1a06404ed..355a37d97 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -125,8 +125,6 @@ grep -q "copying path.*input-0" $TEST_ROOT/log grep -q "copying path.*top" $TEST_ROOT/log -if [ -n "$HAVE_SODIUM" ]; then - # Create a signed binary cache. clearCache clearCacheCache @@ -181,8 +179,6 @@ clearCacheCache nix-store -r $outPath --substituters "file://$cacheDir2 file://$cacheDir" --trusted-public-keys "$publicKey" -fi # HAVE_LIBSODIUM - unset _NIX_FORCE_HTTP diff --git a/tests/common.sh.in b/tests/common.sh.in index 5e00d64f1..5489c0c44 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -34,7 +34,6 @@ coreutils=@coreutils@ export dot=@dot@ export SHELL="@bash@" export PAGER=cat -export HAVE_SODIUM="@HAVE_SODIUM@" export busybox="@sandbox_shell@" export version=@PACKAGE_VERSION@ From 08133503494d023b646b3107acf159a5274466ec Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 7 Jan 2021 21:51:46 +0100 Subject: [PATCH 18/41] Add 'nix store prefetch-{file,tarball}' These replace nix-prefetch-url and nix-prefetch-url --unpack, respectively. --- src/libstore/filetransfer.hh | 2 +- src/nix-prefetch-url/nix-prefetch-url.cc | 232 --------------- src/nix/local.mk | 1 - src/nix/prefetch.cc | 352 +++++++++++++++++++++++ src/nix/store-prefetch-file.md | 32 +++ src/nix/store-prefetch-tarball.md | 31 ++ 6 files changed, 416 insertions(+), 234 deletions(-) delete mode 100644 src/nix-prefetch-url/nix-prefetch-url.cc create mode 100644 src/nix/prefetch.cc create mode 100644 src/nix/store-prefetch-file.md create mode 100644 src/nix/store-prefetch-tarball.md diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index afc7e7aa6..45d9ccf89 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -63,7 +63,7 @@ struct FileTransferRequest std::string mimeType; std::function dataCallback; - FileTransferRequest(const std::string & uri) + FileTransferRequest(std::string_view uri) : uri(uri), parentAct(getCurActivity()) { } std::string verb() diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc deleted file mode 100644 index 3bdee55a7..000000000 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ /dev/null @@ -1,232 +0,0 @@ -#include "hash.hh" -#include "shared.hh" -#include "filetransfer.hh" -#include "store-api.hh" -#include "eval.hh" -#include "eval-inline.hh" -#include "common-eval-args.hh" -#include "attr-path.hh" -#include "finally.hh" -#include "../nix/legacy.hh" -#include "progress-bar.hh" -#include "tarfile.hh" - -#include - -#include -#include -#include - -using namespace nix; - - -/* If ‘uri’ starts with ‘mirror://’, then resolve it using the list of - mirrors defined in Nixpkgs. */ -string resolveMirrorUri(EvalState & state, string uri) -{ - if (string(uri, 0, 9) != "mirror://") return uri; - - string s(uri, 9); - auto p = s.find('/'); - if (p == string::npos) throw Error("invalid mirror URI"); - string mirrorName(s, 0, p); - - Value vMirrors; - state.eval(state.parseExprFromString("import ", "."), vMirrors); - state.forceAttrs(vMirrors); - - auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName)); - if (mirrorList == vMirrors.attrs->end()) - throw Error("unknown mirror name '%1%'", mirrorName); - state.forceList(*mirrorList->value); - - if (mirrorList->value->listSize() < 1) - throw Error("mirror URI '%1%' did not expand to anything", uri); - - string mirror = state.forceString(*mirrorList->value->listElems()[0]); - return mirror + (hasSuffix(mirror, "/") ? "" : "/") + string(s, p + 1); -} - - -static int main_nix_prefetch_url(int argc, char * * argv) -{ - { - HashType ht = htSHA256; - std::vector args; - bool printPath = getEnv("PRINT_PATH") == "1"; - bool fromExpr = false; - string attrPath; - bool unpack = false; - bool executable = false; - string name; - - struct MyArgs : LegacyArgs, MixEvalArgs - { - using LegacyArgs::LegacyArgs; - }; - - MyArgs myArgs(std::string(baseNameOf(argv[0])), [&](Strings::iterator & arg, const Strings::iterator & end) { - if (*arg == "--help") - showManPage("nix-prefetch-url"); - else if (*arg == "--version") - printVersion("nix-prefetch-url"); - else if (*arg == "--type") { - string s = getArg(*arg, arg, end); - ht = parseHashType(s); - } - else if (*arg == "--print-path") - printPath = true; - else if (*arg == "--attr" || *arg == "-A") { - fromExpr = true; - attrPath = getArg(*arg, arg, end); - } - else if (*arg == "--unpack") - unpack = true; - else if (*arg == "--executable") - executable = true; - else if (*arg == "--name") - name = getArg(*arg, arg, end); - else if (*arg != "" && arg->at(0) == '-') - return false; - else - args.push_back(*arg); - return true; - }); - - myArgs.parseCmdline(argvToStrings(argc, argv)); - - initPlugins(); - - if (args.size() > 2) - throw UsageError("too many arguments"); - - Finally f([]() { stopProgressBar(); }); - - if (isatty(STDERR_FILENO)) - startProgressBar(); - - auto store = openStore(); - auto state = std::make_unique(myArgs.searchPath, store); - - Bindings & autoArgs = *myArgs.getAutoArgs(*state); - - /* If -A is given, get the URI from the specified Nix - expression. */ - string uri; - if (!fromExpr) { - if (args.empty()) - throw UsageError("you must specify a URI"); - uri = args[0]; - } else { - Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0])); - Value vRoot; - state->evalFile(path, vRoot); - Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first); - state->forceAttrs(v); - - /* Extract the URI. */ - auto attr = v.attrs->find(state->symbols.create("urls")); - if (attr == v.attrs->end()) - throw Error("attribute set does not contain a 'urls' attribute"); - state->forceList(*attr->value); - if (attr->value->listSize() < 1) - throw Error("'urls' list is empty"); - uri = state->forceString(*attr->value->listElems()[0]); - - /* Extract the hash mode. */ - attr = v.attrs->find(state->symbols.create("outputHashMode")); - if (attr == v.attrs->end()) - printInfo("warning: this does not look like a fetchurl call"); - else - unpack = state->forceString(*attr->value) == "recursive"; - - /* Extract the name. */ - if (name.empty()) { - attr = v.attrs->find(state->symbols.create("name")); - if (attr != v.attrs->end()) - name = state->forceString(*attr->value); - } - } - - /* Figure out a name in the Nix store. */ - if (name.empty()) - name = baseNameOf(uri); - if (name.empty()) - throw Error("cannot figure out file name for '%1%'", uri); - - /* If an expected hash is given, the file may already exist in - the store. */ - std::optional expectedHash; - Hash hash(ht); - std::optional storePath; - if (args.size() == 2) { - expectedHash = Hash::parseAny(args[1], ht); - const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - storePath = store->makeFixedOutputPath(recursive, *expectedHash, name); - if (store->isValidPath(*storePath)) - hash = *expectedHash; - else - storePath.reset(); - } - - if (!storePath) { - - auto actualUri = resolveMirrorUri(*state, uri); - - AutoDelete tmpDir(createTempDir(), true); - Path tmpFile = (Path) tmpDir + "/tmp"; - - /* Download the file. */ - { - auto mode = 0600; - if (executable) - mode = 0700; - - AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode); - if (!fd) throw SysError("creating temporary file '%s'", tmpFile); - - FdSink sink(fd.get()); - - FileTransferRequest req(actualUri); - req.decompress = false; - getFileTransfer()->download(std::move(req), sink); - } - - /* Optionally unpack the file. */ - if (unpack) { - printInfo("unpacking..."); - Path unpacked = (Path) tmpDir + "/unpacked"; - createDirs(unpacked); - unpackTarfile(tmpFile, unpacked); - - /* If the archive unpacks to a single file/directory, then use - that as the top-level. */ - auto entries = readDirectory(unpacked); - if (entries.size() == 1) - tmpFile = unpacked + "/" + entries[0].name; - else - tmpFile = unpacked; - } - - const auto method = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - - auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash); - storePath = info.path; - assert(info.ca); - hash = getContentAddressHash(*info.ca); - } - - stopProgressBar(); - - if (!printPath) - printInfo("path is '%s'", store->printStorePath(*storePath)); - - std::cout << printHash16or32(hash) << std::endl; - if (printPath) - std::cout << store->printStorePath(*storePath) << std::endl; - - return 0; - } -} - -static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url); diff --git a/src/nix/local.mk b/src/nix/local.mk index f37b73384..23c08fc86 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -12,7 +12,6 @@ nix_SOURCES := \ $(wildcard src/nix-daemon/*.cc) \ $(wildcard src/nix-env/*.cc) \ $(wildcard src/nix-instantiate/*.cc) \ - $(wildcard src/nix-prefetch-url/*.cc) \ $(wildcard src/nix-store/*.cc) \ nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc new file mode 100644 index 000000000..969299489 --- /dev/null +++ b/src/nix/prefetch.cc @@ -0,0 +1,352 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "filetransfer.hh" +#include "finally.hh" +#include "progress-bar.hh" +#include "tarfile.hh" +#include "attr-path.hh" +#include "eval-inline.hh" +#include "legacy.hh" + +#include + +using namespace nix; + +/* If ‘url’ starts with ‘mirror://’, then resolve it using the list of + mirrors defined in Nixpkgs. */ +string resolveMirrorUrl(EvalState & state, string url) +{ + if (url.substr(0, 9) != "mirror://") return url; + + std::string s(url, 9); + auto p = s.find('/'); + if (p == std::string::npos) throw Error("invalid mirror URL '%s'", url); + std::string mirrorName(s, 0, p); + + Value vMirrors; + // FIXME: use nixpkgs flake + state.eval(state.parseExprFromString("import ", "."), vMirrors); + state.forceAttrs(vMirrors); + + auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName)); + if (mirrorList == vMirrors.attrs->end()) + throw Error("unknown mirror name '%s'", mirrorName); + state.forceList(*mirrorList->value); + + if (mirrorList->value->listSize() < 1) + throw Error("mirror URL '%s' did not expand to anything", url); + + auto mirror = state.forceString(*mirrorList->value->listElems()[0]); + return mirror + (hasSuffix(mirror, "/") ? "" : "/") + string(s, p + 1); +} + +std::tuple prefetchFile( + ref store, + std::string_view url, + std::optional name, + HashType hashType, + std::optional expectedHash, + bool unpack, + bool executable) +{ + auto ingestionMethod = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + + /* Figure out a name in the Nix store. */ + if (!name) { + name = baseNameOf(url); + if (name->empty()) + throw Error("cannot figure out file name for '%s'", url); + } + + std::optional storePath; + std::optional hash; + + /* If an expected hash is given, the file may already exist in + the store. */ + if (expectedHash) { + hashType = expectedHash->type; + storePath = store->makeFixedOutputPath(ingestionMethod, *expectedHash, *name); + if (store->isValidPath(*storePath)) + hash = expectedHash; + else + storePath.reset(); + } + + if (!storePath) { + + AutoDelete tmpDir(createTempDir(), true); + Path tmpFile = (Path) tmpDir + "/tmp"; + + /* Download the file. */ + { + auto mode = 0600; + if (executable) + mode = 0700; + + AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode); + if (!fd) throw SysError("creating temporary file '%s'", tmpFile); + + FdSink sink(fd.get()); + + FileTransferRequest req(url); + req.decompress = false; + getFileTransfer()->download(std::move(req), sink); + } + + /* Optionally unpack the file. */ + if (unpack) { + Activity act(*logger, lvlChatty, actUnknown, + fmt("unpacking '%s'", url)); + Path unpacked = (Path) tmpDir + "/unpacked"; + createDirs(unpacked); + unpackTarfile(tmpFile, unpacked); + + /* If the archive unpacks to a single file/directory, then use + that as the top-level. */ + auto entries = readDirectory(unpacked); + if (entries.size() == 1) + tmpFile = unpacked + "/" + entries[0].name; + else + tmpFile = unpacked; + } + + Activity act(*logger, lvlChatty, actUnknown, + fmt("adding '%s' to the store", url)); + + auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash); + storePath = info.path; + assert(info.ca); + hash = getContentAddressHash(*info.ca); + } + + return {storePath.value(), hash.value()}; +} + +static int main_nix_prefetch_url(int argc, char * * argv) +{ + { + HashType ht = htSHA256; + std::vector args; + bool printPath = getEnv("PRINT_PATH") == "1"; + bool fromExpr = false; + string attrPath; + bool unpack = false; + bool executable = false; + std::optional name; + + struct MyArgs : LegacyArgs, MixEvalArgs + { + using LegacyArgs::LegacyArgs; + }; + + MyArgs myArgs(std::string(baseNameOf(argv[0])), [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--help") + showManPage("nix-prefetch-url"); + else if (*arg == "--version") + printVersion("nix-prefetch-url"); + else if (*arg == "--type") { + string s = getArg(*arg, arg, end); + ht = parseHashType(s); + } + else if (*arg == "--print-path") + printPath = true; + else if (*arg == "--attr" || *arg == "-A") { + fromExpr = true; + attrPath = getArg(*arg, arg, end); + } + else if (*arg == "--unpack") + unpack = true; + else if (*arg == "--executable") + executable = true; + else if (*arg == "--name") + name = getArg(*arg, arg, end); + else if (*arg != "" && arg->at(0) == '-') + return false; + else + args.push_back(*arg); + return true; + }); + + myArgs.parseCmdline(argvToStrings(argc, argv)); + + initPlugins(); + + if (args.size() > 2) + throw UsageError("too many arguments"); + + Finally f([]() { stopProgressBar(); }); + + if (isatty(STDERR_FILENO)) + startProgressBar(); + + auto store = openStore(); + auto state = std::make_unique(myArgs.searchPath, store); + + Bindings & autoArgs = *myArgs.getAutoArgs(*state); + + /* If -A is given, get the URL from the specified Nix + expression. */ + string url; + if (!fromExpr) { + if (args.empty()) + throw UsageError("you must specify a URL"); + url = args[0]; + } else { + Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0])); + Value vRoot; + state->evalFile(path, vRoot); + Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first); + state->forceAttrs(v); + + /* Extract the URL. */ + auto attr = v.attrs->find(state->symbols.create("urls")); + if (attr == v.attrs->end()) + throw Error("attribute set does not contain a 'urls' attribute"); + state->forceList(*attr->value); + if (attr->value->listSize() < 1) + throw Error("'urls' list is empty"); + url = state->forceString(*attr->value->listElems()[0]); + + /* Extract the hash mode. */ + attr = v.attrs->find(state->symbols.create("outputHashMode")); + if (attr == v.attrs->end()) + printInfo("warning: this does not look like a fetchurl call"); + else + unpack = state->forceString(*attr->value) == "recursive"; + + /* Extract the name. */ + if (!name) { + attr = v.attrs->find(state->symbols.create("name")); + if (attr != v.attrs->end()) + name = state->forceString(*attr->value); + } + } + + std::optional expectedHash; + if (args.size() == 2) + expectedHash = Hash::parseAny(args[1], ht); + + auto [storePath, hash] = prefetchFile( + store, resolveMirrorUrl(*state, url), name, ht, expectedHash, unpack, executable); + + stopProgressBar(); + + if (!printPath) + printInfo("path is '%s'", store->printStorePath(storePath)); + + std::cout << printHash16or32(hash) << std::endl; + if (printPath) + std::cout << store->printStorePath(storePath) << std::endl; + + return 0; + } +} + +static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url); + +struct CmdStorePrefetch : StoreCommand, MixJSON +{ + std::string url; + bool executable = false; + bool unpack; + std::optional name; + HashType hashType = htSHA256; + std::optional expectedHash; + + CmdStorePrefetch(bool unpack) + : unpack(unpack) + { + addFlag({ + .longName = "name", + .description = "store path name", + .labels = {"name"}, + .handler = {&name} + }); + + addFlag({ + .longName = "expected-hash", + .description = unpack ? "expected NAR hash of the unpacked tarball" : "expected hash of the file", + .labels = {"hash"}, + .handler = {[&](std::string s) { + expectedHash = Hash::parseAny(s, hashType); + }} + }); + + addFlag(Flag::mkHashTypeFlag("hash-type", &hashType)); + + expectArg("url", &url); + } + + Category category() override { return catUtility; } + + void run(ref store) override + { + auto [storePath, hash] = prefetchFile(store, url, name, hashType, expectedHash, unpack, executable); + + if (json) { + auto res = nlohmann::json::object(); + res["storePath"] = store->printStorePath(storePath); + res["hash"] = hash.to_string(SRI, true); + logger->cout(res.dump()); + } else { + notice("Downloaded '%s' to '%s' (hash '%s').", + url, + store->printStorePath(storePath), + hash.to_string(SRI, true)); + } + } +}; + +struct CmdStorePrefetchFile : CmdStorePrefetch +{ + CmdStorePrefetchFile() + : CmdStorePrefetch(false) + { + name = "source"; + + addFlag({ + .longName = "executable", + .description = "make the resulting file executable", + .handler = {&executable, true}, + }); + } + + std::string description() override + { + return "download a file into the Nix store"; + } + + std::string doc() override + { + return + #include "store-prefetch-file.md" + ; + } +}; + +static auto rCmdStorePrefetchFile = registerCommand2({"store", "prefetch-file"}); + +struct CmdStorePrefetchTarball : CmdStorePrefetch +{ + CmdStorePrefetchTarball() + : CmdStorePrefetch(true) + { + name = "source"; + } + + std::string description() override + { + return "download and unpack a tarball into the Nix store"; + } + + std::string doc() override + { + return + #include "store-prefetch-tarball.md" + ; + } +}; + +static auto rCmdStorePrefetchTarball = registerCommand2({"store", "prefetch-tarball"}); diff --git a/src/nix/store-prefetch-file.md b/src/nix/store-prefetch-file.md new file mode 100644 index 000000000..1663b847b --- /dev/null +++ b/src/nix/store-prefetch-file.md @@ -0,0 +1,32 @@ +R""( + +# Examples + +* Download a file to the Nix store: + + ```console + # nix store prefetch-file https://releases.nixos.org/nix/nix-2.3.10/nix-2.3.10.tar.xz + Downloaded 'https://releases.nixos.org/nix/nix-2.3.10/nix-2.3.10.tar.xz' to + '/nix/store/vbdbi42hgnc4h7pyqzp6h2yf77kw93aw-source' (hash + 'sha256-qKheVd5D0BervxMDbt+1hnTKE2aRWC8XCAwc0SeHt6s='). + ``` + +* Download a file and get the SHA-512 hash: + + ```console + # nix store prefetch-file --json --hash-type sha512 \ + https://releases.nixos.org/nix/nix-2.3.10/nix-2.3.10.tar.xz \ + | jq -r .hash + sha512-6XJxfym0TNH9knxeH4ZOvns6wElFy3uahunl2hJgovACCMEMXSy42s69zWVyGJALXTI+86tpDJGlIcAySEKBbA== + ``` + +# Description + +This command downloads the file *url* to the Nix store. It prints out +the resulting store path and the cryptographic hash of the contents of +the file. + +The name component of the store path defaults to the last component of +*url*, but this can be overriden using `--name`. + +)"" diff --git a/src/nix/store-prefetch-tarball.md b/src/nix/store-prefetch-tarball.md new file mode 100644 index 000000000..535d7e022 --- /dev/null +++ b/src/nix/store-prefetch-tarball.md @@ -0,0 +1,31 @@ +R""( + +# Examples + +* Download a tarball and unpack it: + + ```console + # nix store prefetch-tarball https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz + Downloaded 'https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz' + to '/nix/store/sl5vvk8mb4ma1sjyy03kwpvkz50hd22d-source' (hash + 'sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY='). + ``` + +* Download a tarball and unpack it, unless it already exists in the + Nix store: + + ```console + # nix store prefetch-tarball https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz \ + --expected-hash sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY= + ``` + +# Description + +This command downloads a tarball or zip file from *url*, unpacks it, +and adds the unpacked tree to the Nix store. It prints out the +resulting store path and the NAR hash of that store path. + +The name component of the store path defaults to `source`, but this +can be overriden using `--name`. + +)"" From 93f1678ec60bcacfcc857f361b5f63e37c498eb4 Mon Sep 17 00:00:00 2001 From: Danila Fedorin Date: Fri, 8 Jan 2021 01:53:57 +0000 Subject: [PATCH 19/41] Allow Flake inputs to accept boolean and integer attributes I believe that this makes it possible to do things like Git inputs with submodules, but it also likely applies to other input types from libfetchers. --- src/libexpr/flake/flake.cc | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 4f021570c..41c93bcaa 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -120,11 +120,16 @@ static FlakeInput parseFlakeInput(EvalState & state, expectType(state, nString, *attr.value, *attr.pos); input.follows = parseInputPath(attr.value->string.s); } else { - if (attr.value->type() == nString) + if (attr.value->type() == nString) { attrs.emplace(attr.name, attr.value->string.s); - else - throw TypeError("flake input attribute '%s' is %s while a string is expected", + } else if (attr.value->type() == nBool) { + attrs.emplace(attr.name, Explicit{ attr.value->boolean }); + } else if (attr.value->type() == nInt) { + attrs.emplace(attr.name, attr.value->integer); + } else { + throw TypeError("flake input attribute '%s' is %s while a string, boolean, or integer is expected", attr.name, showType(*attr.value)); + } } } catch (Error & e) { e.addTrace(*attr.pos, hintfmt("in flake attribute '%s'", attr.name)); From ba0f841a078402f95cf93693c3749743c3ab6246 Mon Sep 17 00:00:00 2001 From: Danila Fedorin Date: Fri, 8 Jan 2021 03:13:42 +0000 Subject: [PATCH 20/41] Use switch statement instead of sequence of ifs --- src/libexpr/flake/flake.cc | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 41c93bcaa..9f1e4063f 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -120,15 +120,19 @@ static FlakeInput parseFlakeInput(EvalState & state, expectType(state, nString, *attr.value, *attr.pos); input.follows = parseInputPath(attr.value->string.s); } else { - if (attr.value->type() == nString) { - attrs.emplace(attr.name, attr.value->string.s); - } else if (attr.value->type() == nBool) { - attrs.emplace(attr.name, Explicit{ attr.value->boolean }); - } else if (attr.value->type() == nInt) { - attrs.emplace(attr.name, attr.value->integer); - } else { - throw TypeError("flake input attribute '%s' is %s while a string, boolean, or integer is expected", - attr.name, showType(*attr.value)); + switch (attr.value->type()) { + case nString: + attrs.emplace(attr.name, attr.value->string.s); + break; + case nBool: + attrs.emplace(attr.name, Explicit { attr.value->boolean }); + break; + case nInt: + attrs.emplace(attr.name, attr.value->integer); + break; + default: + throw TypeError("flake input attribute '%s' is %s while a string, boolean, or integer is expected", + attr.name, showType(*attr.value)); } } } catch (Error & e) { From 48a9be2aabf6620ceb00caf7c4c917e4e0a81446 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Jan 2021 10:44:55 +0100 Subject: [PATCH 21/41] Remove mkIntFlag --- src/libutil/args.hh | 26 +++++++++++++++----------- src/nix/verify.cc | 10 +++++++++- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 6ed541a32..3e84ac64a 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -68,8 +68,12 @@ protected: , arity(ArityAny) { } - template - Handler(T * dest) + Handler(std::string * dest) + : fun([=](std::vector ss) { *dest = ss[0]; }) + , arity(1) + { } + + Handler(std::optional * dest) : fun([=](std::vector ss) { *dest = ss[0]; }) , arity(1) { } @@ -79,6 +83,15 @@ protected: : fun([=](std::vector ss) { *dest = val; }) , arity(0) { } + + template + Handler(I * dest) + : fun([=](std::vector ss) { + if (!string2Int(ss[0], *dest)) + throw UsageError("'%s' is not an integer", ss[0]); + }) + , arity(1) + { } }; /* Flags. */ @@ -161,15 +174,6 @@ public: }); } - template - void mkIntFlag(char shortName, const std::string & longName, - const std::string & description, I * dest) - { - mkFlag(shortName, longName, description, [=](I n) { - *dest = n; - }); - } - template void mkFlag(char shortName, const std::string & longName, const std::string & description, std::function fun) diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 16d42349f..620109aac 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -20,6 +20,7 @@ struct CmdVerify : StorePathsCommand { mkFlag(0, "no-contents", "do not verify the contents of each store path", &noContents); mkFlag(0, "no-trust", "do not verify whether each store path is trusted", &noTrust); + addFlag({ .longName = "substituter", .shortName = 's', @@ -27,7 +28,14 @@ struct CmdVerify : StorePathsCommand .labels = {"store-uri"}, .handler = {[&](std::string s) { substituterUris.push_back(s); }} }); - mkIntFlag('n', "sigs-needed", "require that each path has at least N valid signatures", &sigsNeeded); + + addFlag({ + .longName = "sigs-needed", + .shortName = 'n', + .description = "require that each path has at least N valid signatures", + .labels = {"n"}, + .handler = {&sigsNeeded} + }); } std::string description() override From 1d4954e73e389d319416bf29e949b4b1cbc9ebd9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Jan 2021 11:40:36 +0100 Subject: [PATCH 22/41] Remove mkFlag integer specialisation --- src/libmain/shared.cc | 16 +++++++++++++--- src/libutil/args.hh | 18 ------------------ 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 6751a3744..223020378 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -211,9 +211,19 @@ LegacyArgs::LegacyArgs(const std::string & programName, }); auto intSettingAlias = [&](char shortName, const std::string & longName, - const std::string & description, const std::string & dest) { - mkFlag(shortName, longName, description, [=](unsigned int n) { - settings.set(dest, std::to_string(n)); + const std::string & description, const std::string & dest) + { + addFlag({ + .longName = longName, + .shortName = shortName, + .description = description, + .labels = {"n"}, + .handler = {[=](std::string s) { + unsigned int n; + if (!string2Int(s, n)) + throw UsageError("'%s' is not an integer", s); + settings.set(dest, std::to_string(n)); + }} }); }; diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 3e84ac64a..c54b0efaf 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -174,24 +174,6 @@ public: }); } - template - void mkFlag(char shortName, const std::string & longName, - const std::string & description, std::function fun) - { - addFlag({ - .longName = longName, - .shortName = shortName, - .description = description, - .labels = {"N"}, - .handler = {[=](std::string s) { - I n; - if (!string2Int(s, n)) - throw UsageError("flag '--%s' requires a integer argument", longName); - fun(n); - }} - }); - } - void expectArgs(ExpectedArg && arg) { expectedArgs.emplace_back(std::move(arg)); From 29a445840a4f01dfb1533806f8dfc28f7dc4bee9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Jan 2021 11:42:44 +0100 Subject: [PATCH 23/41] Remove unused mkFlag1 --- src/libutil/args.hh | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index c54b0efaf..62b9516d8 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -143,19 +143,6 @@ public: /* Helper functions for constructing flags / positional arguments. */ - void mkFlag1(char shortName, const std::string & longName, - const std::string & label, const std::string & description, - std::function fun) - { - addFlag({ - .longName = longName, - .shortName = shortName, - .description = description, - .labels = {label}, - .handler = {[=](std::string s) { fun(s); }} - }); - } - void mkFlag(char shortName, const std::string & name, const std::string & description, bool * dest) { From 6548b89cc4eb214cb4632fd4332c610f2d1f0a9d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Jan 2021 12:22:21 +0100 Subject: [PATCH 24/41] string2Int(): Return std::optional --- src/libexpr/attr-path.cc | 14 ++++++-------- src/libexpr/get-drvs.cc | 8 ++++---- src/libfetchers/path.cc | 6 +++--- src/libmain/shared.cc | 6 +++--- src/libmain/shared.hh | 7 +++---- src/libstore/build/derivation-goal.cc | 4 +--- src/libstore/globals.cc | 8 ++++++-- src/libstore/local-store.cc | 4 +++- src/libstore/names.cc | 12 ++++++------ src/libstore/nar-info.cc | 8 ++++++-- src/libstore/profiles.cc | 11 +++++------ src/libstore/store-api.cc | 13 +++++++------ src/libutil/args.hh | 4 +++- src/libutil/config.cc | 4 +++- src/libutil/util.hh | 18 ++++++++++++------ src/nix-env/nix-env.cc | 19 +++++++++---------- src/nix/profile.cc | 5 ++--- 17 files changed, 82 insertions(+), 69 deletions(-) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 2d37dcb7e..9dd557205 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -52,9 +52,7 @@ std::pair findAlongAttrPath(EvalState & state, const string & attr for (auto & attr : tokens) { /* Is i an index (integer) or a normal attribute name? */ - enum { apAttr, apIndex } apType = apAttr; - unsigned int attrIndex; - if (string2Int(attr, attrIndex)) apType = apIndex; + auto attrIndex = string2Int(attr); /* Evaluate the expression. */ Value * vNew = state.allocValue(); @@ -65,7 +63,7 @@ std::pair findAlongAttrPath(EvalState & state, const string & attr /* It should evaluate to either a set or an expression, according to what is specified in the attrPath. */ - if (apType == apAttr) { + if (!attrIndex) { if (v->type() != nAttrs) throw TypeError( @@ -82,17 +80,17 @@ std::pair findAlongAttrPath(EvalState & state, const string & attr pos = *a->pos; } - else if (apType == apIndex) { + else { if (!v->isList()) throw TypeError( "the expression selected by the selection path '%1%' should be a list but is %2%", attrPath, showType(*v)); - if (attrIndex >= v->listSize()) - throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath); + if (*attrIndex >= v->listSize()) + throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", *attrIndex, attrPath); - v = v->listElems()[attrIndex]; + v = v->listElems()[*attrIndex]; pos = noPos; } diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 32c115c12..1a3990ea1 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -214,8 +214,8 @@ NixInt DrvInfo::queryMetaInt(const string & name, NixInt def) if (v->type() == nString) { /* Backwards compatibility with before we had support for integer meta fields. */ - NixInt n; - if (string2Int(v->string.s, n)) return n; + if (auto n = string2Int(v->string.s)) + return *n; } return def; } @@ -228,8 +228,8 @@ NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def) if (v->type() == nString) { /* Backwards compatibility with before we had support for float meta fields. */ - NixFloat n; - if (string2Float(v->string.s, n)) return n; + if (auto n = string2Float(v->string.s)) + return *n; } return def; } diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index bcb904c0d..d1003de57 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -20,10 +20,10 @@ struct PathInputScheme : InputScheme if (name == "rev" || name == "narHash") input.attrs.insert_or_assign(name, value); else if (name == "revCount" || name == "lastModified") { - uint64_t n; - if (!string2Int(value, n)) + if (auto n = string2Int(value)) + input.attrs.insert_or_assign(name, *n); + else throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); - input.attrs.insert_or_assign(name, n); } else throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 223020378..f1feeddd6 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -219,10 +219,10 @@ LegacyArgs::LegacyArgs(const std::string & programName, .description = description, .labels = {"n"}, .handler = {[=](std::string s) { - unsigned int n; - if (!string2Int(s, n)) + if (auto n = string2Int(s)) + settings.set(dest, std::to_string(*n)); + else throw UsageError("'%s' is not an integer", s); - settings.set(dest, std::to_string(n)); }} }); }; diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index ffae5d796..38f627b44 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -70,10 +70,9 @@ template N getIntArg(const string & opt, s.resize(s.size() - 1); } } - N n; - if (!string2Int(s, n)) - throw UsageError("'%1%' requires an integer argument", opt); - return n * multiplier; + if (auto n = string2Int(s)) + return *n * multiplier; + throw UsageError("'%1%' requires an integer argument", opt); } diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index af3ab87a9..35f365795 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1699,12 +1699,10 @@ void DerivationGoal::startBuilder() userNamespaceSync.writeSide = -1; }); - pid_t tmp; auto ss = tokenizeString>(readLine(builderOut.readSide.get())); assert(ss.size() == 2); usingUserNamespace = ss[0] == "1"; - if (!string2Int(ss[1], tmp)) abort(); - pid = tmp; + pid = string2Int(ss[1]).value(); if (usingUserNamespace) { /* Set the UID/GID mapping of the builder's user namespace diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index ad66ef8a8..0531aad9f 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -228,8 +228,12 @@ template<> void BaseSetting::convertToArg(Args & args, const std::s void MaxBuildJobsSetting::set(const std::string & str, bool append) { if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency()); - else if (!string2Int(str, value)) - throw UsageError("configuration setting '%s' should be 'auto' or an integer", name); + else { + if (auto n = string2Int(str)) + value = *n; + else + throw UsageError("configuration setting '%s' should be 'auto' or an integer", name); + } } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 702e7b136..c61f34275 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -66,8 +66,10 @@ int getSchema(Path schemaPath) int curSchema = 0; if (pathExists(schemaPath)) { string s = readFile(schemaPath); - if (!string2Int(s, curSchema)) + auto n = string2Int(s); + if (!n) throw Error("'%1%' is corrupt", schemaPath); + curSchema = *n; } return curSchema; } diff --git a/src/libstore/names.cc b/src/libstore/names.cc index 41e28dc99..ce808accc 100644 --- a/src/libstore/names.cc +++ b/src/libstore/names.cc @@ -80,16 +80,16 @@ string nextComponent(string::const_iterator & p, static bool componentsLT(const string & c1, const string & c2) { - int n1, n2; - bool c1Num = string2Int(c1, n1), c2Num = string2Int(c2, n2); + auto n1 = string2Int(c1); + auto n2 = string2Int(c2); - if (c1Num && c2Num) return n1 < n2; - else if (c1 == "" && c2Num) return true; + if (n1 && n2) return *n1 < *n2; + else if (c1 == "" && n2) return true; else if (c1 == "pre" && c2 != "pre") return true; else if (c2 == "pre") return false; /* Assume that `2.3a' < `2.3.1'. */ - else if (c2Num) return true; - else if (c1Num) return false; + else if (n2) return true; + else if (n1) return false; else return c1 < c2; } diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 3454f34bb..49079388a 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -46,14 +46,18 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "FileHash") fileHash = parseHashField(value); else if (name == "FileSize") { - if (!string2Int(value, fileSize)) throw corrupt(); + auto n = string2Int(value); + if (!n) throw corrupt(); + fileSize = *n; } else if (name == "NarHash") { narHash = parseHashField(value); haveNarHash = true; } else if (name == "NarSize") { - if (!string2Int(value, narSize)) throw corrupt(); + auto n = string2Int(value); + if (!n) throw corrupt(); + narSize = *n; } else if (name == "References") { auto refs = tokenizeString(value, " "); diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index ed10dd519..5d1723886 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -21,9 +21,8 @@ static std::optional parseName(const string & profileName, con string s = string(name, profileName.size() + 1); string::size_type p = s.find("-link"); if (p == string::npos) return {}; - unsigned int n; - if (string2Int(string(s, 0, p), n) && n >= 0) - return n; + if (auto n = string2Int(s.substr(0, p))) + return *n; else return {}; } @@ -214,12 +213,12 @@ void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, b { time_t curTime = time(0); string strDays = string(timeSpec, 0, timeSpec.size() - 1); - int days; + auto days = string2Int(strDays); - if (!string2Int(strDays, days) || days < 1) + if (!days || *days < 1) throw Error("invalid number of days specifier '%1%'", timeSpec); - time_t oldTime = curTime - days * 24 * 3600; + time_t oldTime = curTime - *days * 24 * 3600; deleteGenerationsOlderThan(profile, oldTime, dryRun); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 7aca22bde..01e2fcc7b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -932,19 +932,20 @@ std::optional decodeValidPathInfo(const Store & store, std::istre getline(str, s); auto narHash = Hash::parseAny(s, htSHA256); getline(str, s); - uint64_t narSize; - if (!string2Int(s, narSize)) throw Error("number expected"); - hashGiven = { narHash, narSize }; + auto narSize = string2Int(s); + if (!narSize) throw Error("number expected"); + hashGiven = { narHash, *narSize }; } ValidPathInfo info(store.parseStorePath(path), hashGiven->first); info.narSize = hashGiven->second; std::string deriver; getline(str, deriver); if (deriver != "") info.deriver = store.parseStorePath(deriver); - string s; int n; + string s; getline(str, s); - if (!string2Int(s, n)) throw Error("number expected"); - while (n--) { + auto n = string2Int(s); + if (!n) throw Error("number expected"); + while ((*n)--) { getline(str, s); info.references.insert(store.parseStorePath(s)); } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 62b9516d8..823d843aa 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -87,7 +87,9 @@ protected: template Handler(I * dest) : fun([=](std::vector ss) { - if (!string2Int(ss[0], *dest)) + if (auto n = string2Int(ss[0])) + *dest = *n; + else throw UsageError("'%s' is not an integer", ss[0]); }) , arity(1) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 7af3e7883..7467e5ac0 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -230,7 +230,9 @@ template void BaseSetting::set(const std::string & str, bool append) { static_assert(std::is_integral::value, "Integer required."); - if (!string2Int(str, value)) + if (auto n = string2Int(str)) + value = *n; + else throw UsageError("setting '%s' has invalid value '%s'", name, str); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 0f82bed78..7a4d5fe92 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -397,21 +397,27 @@ bool statusOk(int status); /* Parse a string into an integer. */ -template bool string2Int(const string & s, N & n) +template +std::optional string2Int(const std::string & s) { - if (string(s, 0, 1) == "-" && !std::numeric_limits::is_signed) - return false; + if (s.substr(0, 1) == "-" && !std::numeric_limits::is_signed) + return {}; std::istringstream str(s); + N n; str >> n; - return str && str.get() == EOF; + if (str && str.get() == EOF) return n; + return {}; } /* Parse a string into a float. */ -template bool string2Float(const string & s, N & n) +template +std::optional string2Float(const string & s) { std::istringstream str(s); + N n; str >> n; - return str && str.get() == EOF; + if (str && str.get() == EOF) return n; + return {}; } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 6c2e075ed..9963f05d9 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1250,11 +1250,10 @@ static void opSwitchGeneration(Globals & globals, Strings opFlags, Strings opArg if (opArgs.size() != 1) throw UsageError("exactly one argument expected"); - GenerationNumber dstGen; - if (!string2Int(opArgs.front(), dstGen)) + if (auto dstGen = string2Int(opArgs.front())) + switchGeneration(globals, *dstGen); + else throw UsageError("expected a generation number"); - - switchGeneration(globals, dstGen); } @@ -1308,17 +1307,17 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr if(opArgs.front().size() < 2) throw Error("invalid number of generations ‘%1%’", opArgs.front()); string str_max = string(opArgs.front(), 1, opArgs.front().size()); - GenerationNumber max; - if (!string2Int(str_max, max) || max == 0) + auto max = string2Int(str_max); + if (!max || *max == 0) throw Error("invalid number of generations to keep ‘%1%’", opArgs.front()); - deleteGenerationsGreaterThan(globals.profile, max, globals.dryRun); + deleteGenerationsGreaterThan(globals.profile, *max, globals.dryRun); } else { std::set gens; for (auto & i : opArgs) { - GenerationNumber n; - if (!string2Int(i, n)) + if (auto n = string2Int(i)) + gens.insert(*n); + else throw UsageError("invalid generation number '%1%'", i); - gens.insert(n); } deleteGenerations(globals.profile, gens, globals.dryRun); } diff --git a/src/nix/profile.cc b/src/nix/profile.cc index d8d2b3a70..8cdd34a20 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -209,9 +209,8 @@ public: std::vector res; for (auto & s : _matchers) { - size_t n; - if (string2Int(s, n)) - res.push_back(n); + if (auto n = string2Int(s)) + res.push_back(*n); else if (store->isStorePath(s)) res.push_back(s); else From 17beae299d5e6bb511c453d0b9d0d7ef906b3d14 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Jan 2021 12:51:19 +0100 Subject: [PATCH 25/41] Support binary unit prefixes in command line arguments --- src/libmain/shared.cc | 6 ++---- src/libmain/shared.hh | 17 +---------------- src/libutil/args.hh | 5 +---- src/libutil/util.hh | 28 +++++++++++++++++++++++++--- 4 files changed, 29 insertions(+), 27 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index f1feeddd6..e797c2fb9 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -219,10 +219,8 @@ LegacyArgs::LegacyArgs(const std::string & programName, .description = description, .labels = {"n"}, .handler = {[=](std::string s) { - if (auto n = string2Int(s)) - settings.set(dest, std::to_string(*n)); - else - throw UsageError("'%s' is not an integer", s); + auto n = string2IntWithUnitPrefix(s); + settings.set(dest, std::to_string(n)); }} }); }; diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 38f627b44..edc7b5efa 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -57,22 +57,7 @@ template N getIntArg(const string & opt, { ++i; if (i == end) throw UsageError("'%1%' requires an argument", opt); - string s = *i; - N multiplier = 1; - if (allowUnit && !s.empty()) { - char u = std::toupper(*s.rbegin()); - if (std::isalpha(u)) { - if (u == 'K') multiplier = 1ULL << 10; - else if (u == 'M') multiplier = 1ULL << 20; - else if (u == 'G') multiplier = 1ULL << 30; - else if (u == 'T') multiplier = 1ULL << 40; - else throw UsageError("invalid unit specifier '%1%'", u); - s.resize(s.size() - 1); - } - } - if (auto n = string2Int(s)) - return *n * multiplier; - throw UsageError("'%1%' requires an integer argument", opt); + return string2IntWithUnitPrefix(*i); } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 823d843aa..3783bc84f 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -87,10 +87,7 @@ protected: template Handler(I * dest) : fun([=](std::vector ss) { - if (auto n = string2Int(ss[0])) - *dest = *n; - else - throw UsageError("'%s' is not an integer", ss[0]); + *dest = string2IntWithUnitPrefix(ss[0]); }) , arity(1) { } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 7a4d5fe92..ab0bd865a 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -401,12 +401,34 @@ template std::optional string2Int(const std::string & s) { if (s.substr(0, 1) == "-" && !std::numeric_limits::is_signed) - return {}; + return std::nullopt; std::istringstream str(s); N n; str >> n; if (str && str.get() == EOF) return n; - return {}; + return std::nullopt; +} + +/* Like string2Int(), but support an optional suffix 'K', 'M', 'G' or + 'T' denoting a binary unit prefix. */ +template +N string2IntWithUnitPrefix(std::string s) +{ + N multiplier = 1; + if (!s.empty()) { + char u = std::toupper(*s.rbegin()); + if (std::isalpha(u)) { + if (u == 'K') multiplier = 1ULL << 10; + else if (u == 'M') multiplier = 1ULL << 20; + else if (u == 'G') multiplier = 1ULL << 30; + else if (u == 'T') multiplier = 1ULL << 40; + else throw UsageError("invalid unit specifier '%1%'", u); + s.resize(s.size() - 1); + } + } + if (auto n = string2Int(s)) + return *n * multiplier; + throw UsageError("'%s' is not an integer", s); } /* Parse a string into a float. */ @@ -417,7 +439,7 @@ std::optional string2Float(const string & s) N n; str >> n; if (str && str.get() == EOF) return n; - return {}; + return std::nullopt; } From e21aee58f6dd7785df50d5d2a473feb5f6b2ed4f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Jan 2021 14:17:06 +0100 Subject: [PATCH 26/41] Fix tests --- src/libutil/tests/tests.cc | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index 35a5d27bb..58df9c5ac 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -320,20 +320,15 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(string2Float, emptyString) { - double n; - ASSERT_EQ(string2Float("", n), false); + ASSERT_EQ(string2Float(""), std::nullopt); } TEST(string2Float, trivialConversions) { - double n; - ASSERT_EQ(string2Float("1.0", n), true); - ASSERT_EQ(n, 1.0); + ASSERT_EQ(string2Float("1.0"), 1.0); - ASSERT_EQ(string2Float("0.0", n), true); - ASSERT_EQ(n, 0.0); + ASSERT_EQ(string2Float("0.0"), 0.0); - ASSERT_EQ(string2Float("-100.25", n), true); - ASSERT_EQ(n, (-100.25)); + ASSERT_EQ(string2Float("-100.25"), -100.25); } /* ---------------------------------------------------------------------------- @@ -341,20 +336,15 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(string2Int, emptyString) { - double n; - ASSERT_EQ(string2Int("", n), false); + ASSERT_EQ(string2Int(""), std::nullopt); } TEST(string2Int, trivialConversions) { - double n; - ASSERT_EQ(string2Int("1", n), true); - ASSERT_EQ(n, 1); + ASSERT_EQ(string2Int("1"), 1); - ASSERT_EQ(string2Int("0", n), true); - ASSERT_EQ(n, 0); + ASSERT_EQ(string2Int("0"), 0); - ASSERT_EQ(string2Int("-100", n), true); - ASSERT_EQ(n, (-100)); + ASSERT_EQ(string2Int("-100"), -100); } /* ---------------------------------------------------------------------------- From 1db3f84baccc30ac38227c1f7edc3bfbc8e5ff5b Mon Sep 17 00:00:00 2001 From: Danila Date: Fri, 8 Jan 2021 16:12:21 -0800 Subject: [PATCH 27/41] Upcase "Boolean" in Flake attribute type error Co-authored-by: Eelco Dolstra --- src/libexpr/flake/flake.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 9f1e4063f..61aeae543 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -131,7 +131,7 @@ static FlakeInput parseFlakeInput(EvalState & state, attrs.emplace(attr.name, attr.value->integer); break; default: - throw TypeError("flake input attribute '%s' is %s while a string, boolean, or integer is expected", + throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", attr.name, showType(*attr.value)); } } From fdcd62eec59485665b919c048874de05235b5971 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 10 Jan 2021 23:20:02 +0100 Subject: [PATCH 28/41] Add 'nix store gc' command --- src/nix/store-gc.cc | 43 +++++++++++++++++++++++++++++++++++++++++++ src/nix/store-gc.md | 21 +++++++++++++++++++++ tests/flakes.sh | 6 +++--- 3 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 src/nix/store-gc.cc create mode 100644 src/nix/store-gc.md diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc new file mode 100644 index 000000000..6e9607d03 --- /dev/null +++ b/src/nix/store-gc.cc @@ -0,0 +1,43 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdStoreGC : StoreCommand, MixDryRun +{ + GCOptions options; + + CmdStoreGC() + { + addFlag({ + .longName = "max", + .description = "stop after freeing `n` bytes of disk space", + .labels = {"n"}, + .handler = {&options.maxFreed} + }); + } + + std::string description() override + { + return "perform garbage collection on a Nix store"; + } + + std::string doc() override + { + return + #include "store-gc.md" + ; + } + + void run(ref store) override + { + options.action = dryRun ? GCOptions::gcReturnDead : GCOptions::gcDeleteDead; + GCResults results; + PrintFreed freed(options.action == GCOptions::gcDeleteDead, results); + store->collectGarbage(options, results); + } +}; + +static auto rCmdStoreGC = registerCommand2({"store", "gc"}); diff --git a/src/nix/store-gc.md b/src/nix/store-gc.md new file mode 100644 index 000000000..956b3c872 --- /dev/null +++ b/src/nix/store-gc.md @@ -0,0 +1,21 @@ +R""( + +# Examples + +* Delete unreachable paths in the Nix store: + + ```console + # nix store gc + ``` + +* Delete up to 1 gigabyte of garbage: + + ```console + # nix store gc --max 1G + ``` + +# Description + +This command deletes unreachable paths in the Nix store. + +)"" diff --git a/tests/flakes.sh b/tests/flakes.sh index 5aec563ac..2b7bcdd68 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -276,18 +276,18 @@ git -C $flake3Dir commit -m 'Add lockfile' # Test whether registry caching works. nix registry list --flake-registry file://$registry | grep -q flake3 mv $registry $registry.tmp -nix-store --gc +nix store gc nix registry list --flake-registry file://$registry --refresh | grep -q flake3 mv $registry.tmp $registry # Test whether flakes are registered as GC roots for offline use. # FIXME: use tarballs rather than git. rm -rf $TEST_HOME/.cache -nix-store --gc # get rid of copies in the store to ensure they get fetched to our git cache +nix store gc # get rid of copies in the store to ensure they get fetched to our git cache _NIX_FORCE_HTTP=1 nix build -o $TEST_ROOT/result git+file://$flake2Dir#bar mv $flake1Dir $flake1Dir.tmp mv $flake2Dir $flake2Dir.tmp -nix-store --gc +nix store gc _NIX_FORCE_HTTP=1 nix build -o $TEST_ROOT/result git+file://$flake2Dir#bar _NIX_FORCE_HTTP=1 nix build -o $TEST_ROOT/result git+file://$flake2Dir#bar --refresh mv $flake1Dir.tmp $flake1Dir From 93ad6430edf3d7efa5948d1e0ca0447e4666b121 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 Jan 2021 12:36:39 +0100 Subject: [PATCH 29/41] nix store prefetch-tarball -> nix flake prefetch --- src/nix/flake-prefetch.md | 28 +++++++++++ src/nix/flake.cc | 40 ++++++++++++++++ src/nix/prefetch.cc | 77 +++++++++---------------------- src/nix/store-prefetch-tarball.md | 31 ------------- 4 files changed, 89 insertions(+), 87 deletions(-) create mode 100644 src/nix/flake-prefetch.md delete mode 100644 src/nix/store-prefetch-tarball.md diff --git a/src/nix/flake-prefetch.md b/src/nix/flake-prefetch.md new file mode 100644 index 000000000..a1cf0289a --- /dev/null +++ b/src/nix/flake-prefetch.md @@ -0,0 +1,28 @@ +R""( + +# Examples + +* Download a tarball and unpack it: + + ```console + # nix flake prefetch https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz + Downloaded 'https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz?narHash=sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY=' + to '/nix/store/sl5vvk8mb4ma1sjyy03kwpvkz50hd22d-source' (hash + 'sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY='). + ``` + +* Download the `dwarffs` flake (looked up in the flake registry): + + ```console + # nix flake prefetch dwarffs --json + {"hash":"sha256-VHg3MYVgQ12LeRSU2PSoDeKlSPD8PYYEFxxwkVVDRd0=" + ,"storePath":"/nix/store/hang3792qwdmm2n0d9nsrs5n6bsws6kv-source"} + ``` + +# Description + +This command downloads the source tree denoted by flake reference +*flake-url*. Note that this does not need to be a flake (i.e. it does +not have to contain a `flake.nix` file). + +)"" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 2b91faa64..b73b9cf4e 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -960,6 +960,45 @@ struct CmdFlakeShow : FlakeCommand } }; +struct CmdFlakePrefetch : FlakeCommand, MixJSON +{ + CmdFlakePrefetch() + { + } + + std::string description() override + { + return "download the source tree denoted by a flake reference into the Nix store"; + } + + std::string doc() override + { + return + #include "flake-prefetch.md" + ; + } + + void run(ref store) override + { + auto originalRef = getFlakeRef(); + auto resolvedRef = originalRef.resolve(store); + auto [tree, lockedRef] = resolvedRef.fetchTree(store); + auto hash = store->queryPathInfo(tree.storePath)->narHash; + + if (json) { + auto res = nlohmann::json::object(); + res["storePath"] = store->printStorePath(tree.storePath); + res["hash"] = hash.to_string(SRI, true); + logger->cout(res.dump()); + } else { + notice("Downloaded '%s' to '%s' (hash '%s').", + lockedRef.to_string(), + store->printStorePath(tree.storePath), + hash.to_string(SRI, true)); + } + } +}; + struct CmdFlake : NixMultiCommand { CmdFlake() @@ -973,6 +1012,7 @@ struct CmdFlake : NixMultiCommand {"clone", []() { return make_ref(); }}, {"archive", []() { return make_ref(); }}, {"show", []() { return make_ref(); }}, + {"prefetch", []() { return make_ref(); }}, }) { } diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 969299489..ce8c85ecf 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -246,17 +246,15 @@ static int main_nix_prefetch_url(int argc, char * * argv) static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url); -struct CmdStorePrefetch : StoreCommand, MixJSON +struct CmdStorePrefetchFile : StoreCommand, MixJSON { std::string url; bool executable = false; - bool unpack; std::optional name; HashType hashType = htSHA256; std::optional expectedHash; - CmdStorePrefetch(bool unpack) - : unpack(unpack) + CmdStorePrefetchFile() { addFlag({ .longName = "name", @@ -267,7 +265,7 @@ struct CmdStorePrefetch : StoreCommand, MixJSON addFlag({ .longName = "expected-hash", - .description = unpack ? "expected NAR hash of the unpacked tarball" : "expected hash of the file", + .description = "expected hash of the file", .labels = {"hash"}, .handler = {[&](std::string s) { expectedHash = Hash::parseAny(s, hashType); @@ -276,14 +274,31 @@ struct CmdStorePrefetch : StoreCommand, MixJSON addFlag(Flag::mkHashTypeFlag("hash-type", &hashType)); + addFlag({ + .longName = "executable", + .description = "make the resulting file executable", + .handler = {&executable, true}, + }); + expectArg("url", &url); } Category category() override { return catUtility; } + std::string description() override + { + return "download a file into the Nix store"; + } + + std::string doc() override + { + return + #include "store-prefetch-file.md" + ; + } void run(ref store) override { - auto [storePath, hash] = prefetchFile(store, url, name, hashType, expectedHash, unpack, executable); + auto [storePath, hash] = prefetchFile(store, url, name, hashType, expectedHash, false, executable); if (json) { auto res = nlohmann::json::object(); @@ -299,54 +314,4 @@ struct CmdStorePrefetch : StoreCommand, MixJSON } }; -struct CmdStorePrefetchFile : CmdStorePrefetch -{ - CmdStorePrefetchFile() - : CmdStorePrefetch(false) - { - name = "source"; - - addFlag({ - .longName = "executable", - .description = "make the resulting file executable", - .handler = {&executable, true}, - }); - } - - std::string description() override - { - return "download a file into the Nix store"; - } - - std::string doc() override - { - return - #include "store-prefetch-file.md" - ; - } -}; - static auto rCmdStorePrefetchFile = registerCommand2({"store", "prefetch-file"}); - -struct CmdStorePrefetchTarball : CmdStorePrefetch -{ - CmdStorePrefetchTarball() - : CmdStorePrefetch(true) - { - name = "source"; - } - - std::string description() override - { - return "download and unpack a tarball into the Nix store"; - } - - std::string doc() override - { - return - #include "store-prefetch-tarball.md" - ; - } -}; - -static auto rCmdStorePrefetchTarball = registerCommand2({"store", "prefetch-tarball"}); diff --git a/src/nix/store-prefetch-tarball.md b/src/nix/store-prefetch-tarball.md deleted file mode 100644 index 535d7e022..000000000 --- a/src/nix/store-prefetch-tarball.md +++ /dev/null @@ -1,31 +0,0 @@ -R""( - -# Examples - -* Download a tarball and unpack it: - - ```console - # nix store prefetch-tarball https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz - Downloaded 'https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz' - to '/nix/store/sl5vvk8mb4ma1sjyy03kwpvkz50hd22d-source' (hash - 'sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY='). - ``` - -* Download a tarball and unpack it, unless it already exists in the - Nix store: - - ```console - # nix store prefetch-tarball https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz \ - --expected-hash sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY= - ``` - -# Description - -This command downloads a tarball or zip file from *url*, unpacks it, -and adds the unpacked tree to the Nix store. It prints out the -resulting store path and the NAR hash of that store path. - -The name component of the store path defaults to `source`, but this -can be overriden using `--name`. - -)"" From 77c9ceda4be8dd304b07f654d8c059a01d839108 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 Jan 2021 19:42:24 +0100 Subject: [PATCH 30/41] Tweak --- doc/manual/src/command-ref/nix-store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md index 827adbd05..361c20cc9 100644 --- a/doc/manual/src/command-ref/nix-store.md +++ b/doc/manual/src/command-ref/nix-store.md @@ -226,7 +226,7 @@ control what gets deleted and in what order: or TiB units. The behaviour of the collector is also influenced by the -`keep-outputs` and `keep-derivations` variables in the Nix +`keep-outputs` and `keep-derivations` settings in the Nix configuration file. By default, the collector prints the total number of freed bytes when it From 6254b1f5d298ff73127d7b0f0da48f142bdc753c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 Jan 2021 19:46:17 +0100 Subject: [PATCH 31/41] Add 'nix store delete' command --- src/nix/store-delete.cc | 45 +++++++++++++++++++++++++++++++++++++++ src/nix/store-delete.md | 24 +++++++++++++++++++++ tests/multiple-outputs.sh | 2 +- 3 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 src/nix/store-delete.cc create mode 100644 src/nix/store-delete.md diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc new file mode 100644 index 000000000..f3677763c --- /dev/null +++ b/src/nix/store-delete.cc @@ -0,0 +1,45 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdStoreDelete : StorePathsCommand +{ + GCOptions options { .action = GCOptions::gcDeleteSpecific }; + + CmdStoreDelete() + { + addFlag({ + .longName = "ignore-liveness", + .description = "do not check whether the paths are reachable from a root", + .handler = {&options.ignoreLiveness, true} + }); + } + + std::string description() override + { + return "delete paths from the Nix store"; + } + + std::string doc() override + { + return + #include "store-delete.md" + ; + } + + void run(ref store, std::vector storePaths) override + { + + for (auto & path : storePaths) + options.pathsToDelete.insert(path); + + GCResults results; + PrintFreed freed(true, results); + store->collectGarbage(options, results); + } +}; + +static auto rCmdStoreDelete = registerCommand2({"store", "delete"}); diff --git a/src/nix/store-delete.md b/src/nix/store-delete.md new file mode 100644 index 000000000..db535f87c --- /dev/null +++ b/src/nix/store-delete.md @@ -0,0 +1,24 @@ +R""( + +# Examples + +* Delete a specific store path: + + ```console + # nix store delete /nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10 + ``` + +# Description + +This command deletes the store paths specified by *installables*. , +but only if it is safe to do so; that is, when the path is not +reachable from a root of the garbage collector. This means that you +can only delete paths that would also be deleted by `nix store +gc`. Thus, `nix store delete` is a more targeted version of `nix store +gc`. + +With the option `--ignore-liveness`, reachability from the roots is +ignored. However, the path still won't be deleted if there are other +paths in the store that refer to it (i.e., depend on it). + +)"" diff --git a/tests/multiple-outputs.sh b/tests/multiple-outputs.sh index 7a6ec181d..de573d4fa 100644 --- a/tests/multiple-outputs.sh +++ b/tests/multiple-outputs.sh @@ -58,7 +58,7 @@ outPath2=$(nix-build $(nix-instantiate multiple-outputs.nix -A a.second) --no-ou # Delete one of the outputs and rebuild it. This will cause a hash # rewrite. -nix-store --delete $TEST_ROOT/result-second --ignore-liveness +nix store delete $TEST_ROOT/result-second --ignore-liveness nix-build multiple-outputs.nix -A a.all -o $TEST_ROOT/result [ "$(cat $TEST_ROOT/result-second/file)" = "second" ] [ "$(cat $TEST_ROOT/result-second/link/file)" = "first" ] From 44fd7a05b655315fa0e6156ac33a1c5624460968 Mon Sep 17 00:00:00 2001 From: Rickard Nilsson Date: Tue, 12 Jan 2021 01:28:00 +0100 Subject: [PATCH 32/41] Don't let 'preferLocalBuild' override 'max-jobs=0' This resolves #3810 by changing the behavior of `max-jobs = 0`, so that specifying the option also avoids local building of derivations with the attribute `preferLocalBuild = true`. --- src/libstore/parsed-derivations.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index e7b7202d4..c5c3ae3dc 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -101,6 +101,10 @@ bool ParsedDerivation::canBuildLocally(Store & localStore) const && !drv.isBuiltin()) return false; + if (settings.maxBuildJobs.get() == 0 + && !drv.isBuiltin()) + return false; + for (auto & feature : getRequiredSystemFeatures()) if (!localStore.systemFeatures.get().count(feature)) return false; From f69820417fa65dbfea88a5f4dd0ccb5376015a6b Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 11 Jan 2021 22:05:32 -0600 Subject: [PATCH 33/41] Set kern.curproc_arch_affinity=0 to escape Rosetta By default, once you enter x86_64 Rosetta 2, macOS will try to run everything in x86_64. So an x86_64 Nix will still try to use x86_64 even when system = aarch64-darwin. To avoid this we can set kern.curproc_arch_affinity sysctl. With kern.curproc_arch_affinity=0, we ignore this preference. This is based on how https://opensource.apple.com/source/system_cmds/system_cmds-880.40.5/arch.tproj/arch.c.auto.html works. Completely undocumented, but seems to work! Note, you can verify this works with this impure Nix expression: ``` { a = derivation { name = "a"; system = "aarch64-darwin"; builder = "/bin/sh"; args = [ "-e" (builtins.toFile "builder" '' [ "$(/usr/bin/arch)" = arm64 ] [ "$(/usr/bin/arch -arch x86_64 /bin/sh -c /usr/bin/arch)" = i386 ] [ "$(/usr/bin/arch -arch arm64 /bin/sh -c /usr/bin/arch)" = arm64 ] /usr/bin/touch $out '') ]; }; b = derivation { name = "b"; system = "x86_64-darwin"; builder = "/bin/sh"; args = [ "-e" (builtins.toFile "builder" '' [ "$(/usr/bin/arch)" = i386 ] [ "$(/usr/bin/arch -arch x86_64 /bin/sh -c /usr/bin/arch)" = i386 ] [ "$(/usr/bin/arch -arch arm64 /bin/sh -c /usr/bin/arch)" = arm64 ] /usr/bin/touch $out '') ]; }; } ``` --- src/libstore/build/derivation-goal.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 35f365795..a02ddb950 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -52,6 +52,7 @@ #if __APPLE__ #include +#include #endif #include @@ -2869,6 +2870,10 @@ void DerivationGoal::runChild() throw SysError("failed to initialize builder"); if (drv->platform == "aarch64-darwin") { + // Unset kern.curproc_arch_affinity so we can escape Rosetta + int affinity = 0; + sysctlbyname("kern.curproc_arch_affinity", NULL, NULL, &affinity, sizeof(affinity)); + cpu_type_t cpu = CPU_TYPE_ARM64; posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); } else if (drv->platform == "x86_64-darwin") { From 29007f8bc6ea42ae1f8311f00c0b5e14f04ec9e5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 12 Jan 2021 19:57:05 +0100 Subject: [PATCH 34/41] nix profile info -> nix profile list --- src/nix/{profile-info.md => profile-list.md} | 2 +- src/nix/profile.cc | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) rename src/nix/{profile-info.md => profile-list.md} (98%) diff --git a/src/nix/profile-info.md b/src/nix/profile-list.md similarity index 98% rename from src/nix/profile-info.md rename to src/nix/profile-list.md index a0c04fc8c..5c29c0b02 100644 --- a/src/nix/profile-info.md +++ b/src/nix/profile-list.md @@ -5,7 +5,7 @@ R""( * Show what packages are installed in the default profile: ```console - # nix profile info + # nix profile list 0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.spotify /nix/store/akpdsid105phbbvknjsdh7hl4v3fhjkr-spotify-1.1.46.916.g416cacf1 1 flake:nixpkgs#legacyPackages.x86_64-linux.zoom-us github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.zoom-us /nix/store/89pmjmbih5qpi7accgacd17ybpgp4xfm-zoom-us-5.4.53350.1027 2 flake:blender-bin#defaultPackage.x86_64-linux github:edolstra/nix-warez/d09d7eea893dcb162e89bc67f6dc1ced14abfc27?dir=blender#defaultPackage.x86_64-linux /nix/store/zfgralhqjnam662kqsgq6isjw8lhrflz-blender-bin-2.91.0 diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 8cdd34a20..ac60d336c 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -336,7 +336,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf } }; -struct CmdProfileInfo : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile +struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile { std::string description() override { @@ -346,7 +346,7 @@ struct CmdProfileInfo : virtual EvalCommand, virtual StoreCommand, MixDefaultPro std::string doc() override { return - #include "profile-info.md" + #include "profile-list.md" ; } @@ -408,7 +408,7 @@ struct CmdProfile : NixMultiCommand {"install", []() { return make_ref(); }}, {"remove", []() { return make_ref(); }}, {"upgrade", []() { return make_ref(); }}, - {"info", []() { return make_ref(); }}, + {"list", []() { return make_ref(); }}, {"diff-closures", []() { return make_ref(); }}, }) { } From 2f463e90ed077e066455a9ef6e024b18fd61c4de Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 12 Jan 2021 23:51:07 +0100 Subject: [PATCH 35/41] Add 'nix profile history' command Replaces 'nix-env --list-generations'. Similar to 'nix profile diff-closures' but shows only the changes in top-level packages. --- src/nix/command.hh | 2 + src/nix/profile-history.md | 26 +++++++++ src/nix/profile.cc | 114 +++++++++++++++++++++++++++++++++++++ 3 files changed, 142 insertions(+) create mode 100644 src/nix/profile-history.md diff --git a/src/nix/command.hh b/src/nix/command.hh index 6882db195..3aae57edd 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -261,6 +261,8 @@ void completeFlakeRefWithFragment( const Strings & defaultFlakeAttrPaths, std::string_view prefix); +std::string showVersions(const std::set & versions); + void printClosureDiff( ref store, const StorePath & beforePath, diff --git a/src/nix/profile-history.md b/src/nix/profile-history.md new file mode 100644 index 000000000..d0fe40c82 --- /dev/null +++ b/src/nix/profile-history.md @@ -0,0 +1,26 @@ +R""( + +# Examples + +* Show the changes between each version of your default profile: + + ```console + # nix profile history + Version 508 -> 509: + flake:nixpkgs#legacyPackages.x86_64-linux.awscli: ∅ -> 1.17.13 + + Version 509 -> 510: + flake:nixpkgs#legacyPackages.x86_64-linux.awscli: 1.17.13 -> 1.18.211 + ``` + +# Description + +This command shows what packages were added, removed or upgraded +between subsequent versions of a profile. It only shows top-level +packages, not dependencies; for that, use [`nix profile +diff-closures`](./nix3-profile-diff-closures.md). + +The addition of a package to a profile is denoted by the string `∅ ->` +*version*, whereas the removal is denoted by *version* `-> ∅`. + +)"" diff --git a/src/nix/profile.cc b/src/nix/profile.cc index ac60d336c..ca95817d0 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -8,6 +8,7 @@ #include "flake/flakeref.hh" #include "../nix-env/user-env.hh" #include "profiles.hh" +#include "names.hh" #include #include @@ -21,6 +22,13 @@ struct ProfileElementSource FlakeRef resolvedRef; std::string attrPath; // FIXME: output names + + bool operator < (const ProfileElementSource & other) const + { + return + std::pair(originalRef.to_string(), attrPath) < + std::pair(other.originalRef.to_string(), other.attrPath); + } }; struct ProfileElement @@ -29,6 +37,29 @@ struct ProfileElement std::optional source; bool active = true; // FIXME: priority + + std::string describe() const + { + if (source) + return fmt("%s#%s", source->originalRef, source->attrPath); + StringSet names; + for (auto & path : storePaths) + names.insert(DrvName(path.name()).name); + return concatStringsSep(", ", names); + } + + std::string versions() const + { + StringSet versions; + for (auto & path : storePaths) + versions.insert(DrvName(path.name()).version); + return showVersions(versions); + } + + bool operator < (const ProfileElement & other) const + { + return std::tuple(describe(), storePaths) < std::tuple(other.describe(), other.storePaths); + } }; struct ProfileManifest @@ -142,6 +173,46 @@ struct ProfileManifest return std::move(info.path); } + + static void printDiff(const ProfileManifest & prev, const ProfileManifest & cur, std::string_view indent) + { + auto prevElems = prev.elements; + std::sort(prevElems.begin(), prevElems.end()); + + auto curElems = cur.elements; + std::sort(curElems.begin(), curElems.end()); + + auto i = prevElems.begin(); + auto j = curElems.begin(); + + bool changes = false; + + while (i != prevElems.end() || j != curElems.end()) { + if (j != curElems.end() && (i == prevElems.end() || i->describe() > j->describe())) { + std::cout << fmt("%s%s: ∅ -> %s\n", indent, j->describe(), j->versions()); + changes = true; + ++j; + } + else if (i != prevElems.end() && (j == curElems.end() || i->describe() < j->describe())) { + std::cout << fmt("%s%s: %s -> ∅\n", indent, i->describe(), i->versions()); + changes = true; + ++i; + } + else { + auto v1 = i->versions(); + auto v2 = j->versions(); + if (v1 != v2) { + std::cout << fmt("%s%s: %s -> %s\n", indent, i->describe(), v1, v2); + changes = true; + } + ++i; + ++j; + } + } + + if (!changes) + std::cout << fmt("%sNo changes.\n", indent); + } }; struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile @@ -401,6 +472,48 @@ struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile } }; +struct CmdProfileHistory : virtual StoreCommand, EvalCommand, MixDefaultProfile +{ + std::string description() override + { + return "show all versions of a profile"; + } + + std::string doc() override + { + return + #include "profile-history.md" + ; + } + + void run(ref store) override + { + auto [gens, curGen] = findGenerations(*profile); + + std::optional> prevGen; + bool first = true; + + for (auto & gen : gens) { + ProfileManifest manifest(*getEvalState(), gen.path); + + if (!first) std::cout << "\n"; + first = false; + + if (prevGen) + std::cout << fmt("Version %d -> %d:\n", prevGen->first.number, gen.number); + else + std::cout << fmt("Version %d:\n", gen.number); + + ProfileManifest::printDiff( + prevGen ? prevGen->second : ProfileManifest(), + manifest, + " "); + + prevGen = {gen, std::move(manifest)}; + } + } +}; + struct CmdProfile : NixMultiCommand { CmdProfile() @@ -410,6 +523,7 @@ struct CmdProfile : NixMultiCommand {"upgrade", []() { return make_ref(); }}, {"list", []() { return make_ref(); }}, {"diff-closures", []() { return make_ref(); }}, + {"history", []() { return make_ref(); }}, }) { } From 0ca1a5013269060919393afaa708640f574ab350 Mon Sep 17 00:00:00 2001 From: Rickard Nilsson Date: Wed, 13 Jan 2021 10:13:51 +0100 Subject: [PATCH 36/41] Remove a redundant condition in DerivationGoal::tryLocalBuild() --- src/libstore/build/derivation-goal.cc | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 35f365795..415a55d37 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -679,13 +679,9 @@ void DerivationGoal::tryToBuild() } void DerivationGoal::tryLocalBuild() { - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); - - /* Make sure that we are allowed to start a build. If this - derivation prefers to be done locally, do it even if - maxBuildJobs is 0. */ + /* Make sure that we are allowed to start a build. */ unsigned int curBuilds = worker.getNrLocalBuilds(); - if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) { + if (curBuilds >= settings.maxBuildJobs) { worker.waitForBuildSlot(shared_from_this()); outputLocks.unlock(); return; From 3da9a9241cb9f8c284426c220ea285398d0328dd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Jan 2021 14:18:04 +0100 Subject: [PATCH 37/41] Convert option descriptions to Markdown --- src/libexpr/common-eval-args.cc | 10 +++++----- src/libmain/common-args.cc | 13 ++++++------- src/libmain/common-args.hh | 4 ++-- src/libmain/shared.cc | 20 ++++++++++---------- src/nix/add-to-store.cc | 2 +- src/nix/build.cc | 6 +++--- src/nix/bundle.cc | 6 +++--- src/nix/command.cc | 14 +++++++------- src/nix/copy.cc | 8 ++++---- src/nix/develop.cc | 16 ++++++++-------- src/nix/eval.cc | 6 +++--- src/nix/flake.cc | 8 ++++---- src/nix/hash.cc | 19 +++++++++---------- src/nix/installables.cc | 26 +++++++++++++------------- src/nix/ls.cc | 6 +++--- src/nix/main.cc | 14 +++++++------- src/nix/path-info.cc | 8 ++++---- src/nix/prefetch.cc | 8 +++++--- src/nix/run.cc | 2 +- src/nix/show-derivation.cc | 2 +- src/nix/sigs.cc | 6 +++--- src/nix/store-delete.cc | 2 +- src/nix/store-gc.cc | 2 +- src/nix/upgrade-nix.cc | 4 ++-- src/nix/verify.cc | 8 ++++---- src/nix/why-depends.cc | 2 +- 26 files changed, 111 insertions(+), 111 deletions(-) diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 10c1a6975..ffe782454 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -14,14 +14,14 @@ MixEvalArgs::MixEvalArgs() { addFlag({ .longName = "arg", - .description = "argument to be passed to Nix functions", + .description = "Pass the value *expr* as the argument *name* to Nix functions.", .labels = {"name", "expr"}, .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }} }); addFlag({ .longName = "argstr", - .description = "string-valued argument to be passed to Nix functions", + .description = "Pass the string *string* as the argument *name* to Nix functions.", .labels = {"name", "string"}, .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }}, }); @@ -29,14 +29,14 @@ MixEvalArgs::MixEvalArgs() addFlag({ .longName = "include", .shortName = 'I', - .description = "add a path to the list of locations used to look up `<...>` file names", + .description = "Add *path* to the list of locations used to look up `<...>` file names.", .labels = {"path"}, .handler = {[&](std::string s) { searchPath.push_back(s); }} }); addFlag({ .longName = "impure", - .description = "allow access to mutable paths and repositories", + .description = "Allow access to mutable paths and repositories.", .handler = {[&]() { evalSettings.pureEval = false; }}, @@ -44,7 +44,7 @@ MixEvalArgs::MixEvalArgs() addFlag({ .longName = "override-flake", - .description = "override a flake registry value", + .description = "Override the flake registries, redirecting *original-ref* to *resolved-ref*.", .labels = {"original-ref", "resolved-ref"}, .handler = {[&](std::string _from, std::string _to) { auto from = parseFlakeRef(_from, absPath(".")); diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 3e4e475e5..bd5573e5d 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -10,25 +10,25 @@ MixCommonArgs::MixCommonArgs(const string & programName) addFlag({ .longName = "verbose", .shortName = 'v', - .description = "increase verbosity level", + .description = "Increase the logging verbosity level.", .handler = {[]() { verbosity = (Verbosity) (verbosity + 1); }}, }); addFlag({ .longName = "quiet", - .description = "decrease verbosity level", + .description = "Decrease the logging verbosity level.", .handler = {[]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }}, }); addFlag({ .longName = "debug", - .description = "enable debug output", + .description = "Set the logging verbosity level to 'debug'.", .handler = {[]() { verbosity = lvlDebug; }}, }); addFlag({ .longName = "option", - .description = "set a Nix configuration option (overriding `nix.conf`)", + .description = "Set the Nix configuration setting *name* to *value* (overriding `nix.conf`).", .labels = {"name", "value"}, .handler = {[](std::string name, std::string value) { try { @@ -51,8 +51,7 @@ MixCommonArgs::MixCommonArgs(const string & programName) addFlag({ .longName = "log-format", - .description = "format of log output; `raw`, `internal-json`, `bar` " - "or `bar-with-logs`", + .description = "Set the format of log output; one of `raw`, `internal-json`, `bar` or `bar-with-logs`.", .labels = {"format"}, .handler = {[](std::string format) { setLogFormat(format); }}, }); @@ -60,7 +59,7 @@ MixCommonArgs::MixCommonArgs(const string & programName) addFlag({ .longName = "max-jobs", .shortName = 'j', - .description = "maximum number of parallel builds", + .description = "The maximum number of parallel builds.", .labels = Strings{"jobs"}, .handler = {[=](std::string s) { settings.set("max-jobs", s); diff --git a/src/libmain/common-args.hh b/src/libmain/common-args.hh index a4de3dccf..47f341619 100644 --- a/src/libmain/common-args.hh +++ b/src/libmain/common-args.hh @@ -16,7 +16,7 @@ struct MixDryRun : virtual Args MixDryRun() { - mkFlag(0, "dry-run", "show what this command would do without doing it", &dryRun); + mkFlag(0, "dry-run", "Show what this command would do without doing it.", &dryRun); } }; @@ -26,7 +26,7 @@ struct MixJSON : virtual Args MixJSON() { - mkFlag(0, "json", "produce JSON output", &json); + mkFlag(0, "json", "Produce output in JSON format, suitable for consumption by another program.", &json); } }; diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index e797c2fb9..7e27e95c2 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -186,27 +186,27 @@ LegacyArgs::LegacyArgs(const std::string & programName, addFlag({ .longName = "no-build-output", .shortName = 'Q', - .description = "do not show build output", + .description = "Do not show build output.", .handler = {[&]() {setLogFormat(LogFormat::raw); }}, }); addFlag({ .longName = "keep-failed", .shortName ='K', - .description = "keep temporary directories of failed builds", + .description = "Keep temporary directories of failed builds.", .handler = {&(bool&) settings.keepFailed, true}, }); addFlag({ .longName = "keep-going", .shortName ='k', - .description = "keep going after a build fails", + .description = "Keep going after a build fails.", .handler = {&(bool&) settings.keepGoing, true}, }); addFlag({ .longName = "fallback", - .description = "build from source if substitution fails", + .description = "Build from source if substitution fails.", .handler = {&(bool&) settings.tryFallback, true}, }); @@ -225,19 +225,19 @@ LegacyArgs::LegacyArgs(const std::string & programName, }); }; - intSettingAlias(0, "cores", "maximum number of CPU cores to use inside a build", "cores"); - intSettingAlias(0, "max-silent-time", "number of seconds of silence before a build is killed", "max-silent-time"); - intSettingAlias(0, "timeout", "number of seconds before a build is killed", "timeout"); + intSettingAlias(0, "cores", "Maximum number of CPU cores to use inside a build.", "cores"); + intSettingAlias(0, "max-silent-time", "Number of seconds of silence before a build is killed.", "max-silent-time"); + intSettingAlias(0, "timeout", "Number of seconds before a build is killed.", "timeout"); - mkFlag(0, "readonly-mode", "do not write to the Nix store", + mkFlag(0, "readonly-mode", "Do not write to the Nix store.", &settings.readOnlyMode); - mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'", + mkFlag(0, "no-gc-warning", "Disable warnings about not using `--add-root`.", &gcWarning, false); addFlag({ .longName = "store", - .description = "URI of the Nix store to use", + .description = "The URL of the Nix store to use.", .labels = {"store-uri"}, .handler = {&(std::string&) settings.storeUri}, }); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index ea4bbbab9..2ae042789 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -19,7 +19,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand addFlag({ .longName = "name", .shortName = 'n', - .description = "name component of the store path", + .description = "Override the name component of the store path. It defaults to the base name of *path*.", .labels = {"name"}, .handler = {&namePart}, }); diff --git a/src/nix/build.cc b/src/nix/build.cc index c2974d983..4cb8ade08 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -19,7 +19,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile addFlag({ .longName = "out-link", .shortName = 'o', - .description = "path of the symlink to the build result", + .description = "Use *path* as prefix for the symlinks to the build results. It defaults to `result`.", .labels = {"path"}, .handler = {&outLink}, .completer = completePath @@ -27,13 +27,13 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile addFlag({ .longName = "no-link", - .description = "do not create a symlink to the build result", + .description = "Do not create symlinks to the build results.", .handler = {&outLink, Path("")}, }); addFlag({ .longName = "rebuild", - .description = "rebuild an already built package and compare the result to the existing store paths", + .description = "Rebuild an already built package and compare the result to the existing store paths.", .handler = {&buildMode, bmCheck}, }); } diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 5f558b01e..1789e4598 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -16,7 +16,7 @@ struct CmdBundle : InstallableCommand { addFlag({ .longName = "bundler", - .description = "use custom bundler", + .description = fmt("Use a custom bundler instead of the default (`%s`).", bundler), .labels = {"flake-url"}, .handler = {&bundler}, .completer = {[&](size_t, std::string_view prefix) { @@ -27,7 +27,7 @@ struct CmdBundle : InstallableCommand addFlag({ .longName = "out-link", .shortName = 'o', - .description = "path of the symlink to the build result", + .description = "Override the name of the symlink to the build result. It defaults to the base name of the app.", .labels = {"path"}, .handler = {&outLink}, .completer = completePath @@ -90,7 +90,7 @@ struct CmdBundle : InstallableCommand mkString(*evalState->allocAttr(*arg, evalState->symbols.create("system")), settings.thisSystem.get()); arg->attrs->sort(); - + auto vRes = evalState->allocValue(); evalState->callFunction(*bundler.toValue(*evalState).first, *arg, *vRes, noPos); diff --git a/src/nix/command.cc b/src/nix/command.cc index 596217775..ba58c7d6b 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -65,18 +65,18 @@ StorePathsCommand::StorePathsCommand(bool recursive) if (recursive) addFlag({ .longName = "no-recursive", - .description = "apply operation to specified paths only", + .description = "Apply operation to specified paths only.", .handler = {&this->recursive, false}, }); else addFlag({ .longName = "recursive", .shortName = 'r', - .description = "apply operation to closure of the specified paths", + .description = "Apply operation to closure of the specified paths.", .handler = {&this->recursive, true}, }); - mkFlag(0, "all", "apply operation to the entire store", &all); + mkFlag(0, "all", "Apply the operation to every store path.", &all); } void StorePathsCommand::run(ref store) @@ -133,7 +133,7 @@ MixProfile::MixProfile() { addFlag({ .longName = "profile", - .description = "profile to update", + .description = "The profile to update.", .labels = {"path"}, .handler = {&profile}, .completer = completePath @@ -190,14 +190,14 @@ MixEnvironment::MixEnvironment() : ignoreEnvironment(false) addFlag({ .longName = "ignore-environment", .shortName = 'i', - .description = "clear the entire environment (except those specified with --keep)", + .description = "Clear the entire environment (except those specified with `--keep`).", .handler = {&ignoreEnvironment, true}, }); addFlag({ .longName = "keep", .shortName = 'k', - .description = "keep specified environment variable", + .description = "Keep the environment variable *name*.", .labels = {"name"}, .handler = {[&](std::string s) { keep.insert(s); }}, }); @@ -205,7 +205,7 @@ MixEnvironment::MixEnvironment() : ignoreEnvironment(false) addFlag({ .longName = "unset", .shortName = 'u', - .description = "unset specified environment variable", + .description = "Unset the environment variable *name*.", .labels = {"name"}, .handler = {[&](std::string s) { unset.insert(s); }}, }); diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 2394eb46d..f15031a45 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -21,28 +21,28 @@ struct CmdCopy : StorePathsCommand { addFlag({ .longName = "from", - .description = "URI of the source Nix store", + .description = "URL of the source Nix store.", .labels = {"store-uri"}, .handler = {&srcUri}, }); addFlag({ .longName = "to", - .description = "URI of the destination Nix store", + .description = "URL of the destination Nix store.", .labels = {"store-uri"}, .handler = {&dstUri}, }); addFlag({ .longName = "no-check-sigs", - .description = "do not require that paths are signed by trusted keys", + .description = "Do not require that paths are signed by trusted keys.", .handler = {&checkSigs, NoCheckSigs}, }); addFlag({ .longName = "substitute-on-destination", .shortName = 's', - .description = "whether to try substitutes on the destination store (only supported by SSH)", + .description = "Whether to try substitutes on the destination store (only supported by SSH stores).", .handler = {&substitute, Substitute}, }); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index edd87f246..578258394 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -204,7 +204,7 @@ struct Common : InstallableCommand, MixProfile { addFlag({ .longName = "redirect", - .description = "redirect a store path to a mutable location", + .description = "Redirect a store path to a mutable location.", .labels = {"installable", "outputs-dir"}, .handler = {[&](std::string installable, std::string outputsDir) { redirects.push_back({installable, outputsDir}); @@ -334,7 +334,7 @@ struct CmdDevelop : Common, MixEnvironment addFlag({ .longName = "command", .shortName = 'c', - .description = "command and arguments to be executed instead of an interactive shell", + .description = "Instead of starting an interactive shell, start the specified command and arguments.", .labels = {"command", "args"}, .handler = {[&](std::vector ss) { if (ss.empty()) throw UsageError("--command requires at least one argument"); @@ -344,38 +344,38 @@ struct CmdDevelop : Common, MixEnvironment addFlag({ .longName = "phase", - .description = "phase to run (e.g. `build` or `configure`)", + .description = "The stdenv phase to run (e.g. `build` or `configure`).", .labels = {"phase-name"}, .handler = {&phase}, }); addFlag({ .longName = "configure", - .description = "run the configure phase", + .description = "Run the `configure` phase.", .handler = {&phase, {"configure"}}, }); addFlag({ .longName = "build", - .description = "run the build phase", + .description = "Run the `build` phase.", .handler = {&phase, {"build"}}, }); addFlag({ .longName = "check", - .description = "run the check phase", + .description = "Run the `check` phase.", .handler = {&phase, {"check"}}, }); addFlag({ .longName = "install", - .description = "run the install phase", + .description = "Run the `install` phase.", .handler = {&phase, {"install"}}, }); addFlag({ .longName = "installcheck", - .description = "run the installcheck phase", + .description = "Run the `installcheck` phase.", .handler = {&phase, {"installCheck"}}, }); } diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 321df7495..b5049ac65 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -18,18 +18,18 @@ struct CmdEval : MixJSON, InstallableCommand CmdEval() { - mkFlag(0, "raw", "print strings unquoted", &raw); + mkFlag(0, "raw", "Print strings without quotes or escaping.", &raw); addFlag({ .longName = "apply", - .description = "apply a function to each argument", + .description = "Apply the function *expr* to each argument.", .labels = {"expr"}, .handler = {&apply}, }); addFlag({ .longName = "write-to", - .description = "write a string or attrset of strings to 'path'", + .description = "Write a string or attrset of strings to *path*.", .labels = {"path"}, .handler = {&writeTo}, }); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index b73b9cf4e..4cd7d77a0 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -222,7 +222,7 @@ struct CmdFlakeCheck : FlakeCommand { addFlag({ .longName = "no-build", - .description = "do not build checks", + .description = "Do not build checks.", .handler = {&build, false} }); } @@ -573,7 +573,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand addFlag({ .longName = "template", .shortName = 't', - .description = "the template to use", + .description = "The template to use.", .labels = {"template"}, .handler = {&templateUrl}, .completer = {[&](size_t, std::string_view prefix) { @@ -717,7 +717,7 @@ struct CmdFlakeClone : FlakeCommand addFlag({ .longName = "dest", .shortName = 'f', - .description = "destination path", + .description = "Clone the flake to path *dest*.", .labels = {"path"}, .handler = {&destDir} }); @@ -807,7 +807,7 @@ struct CmdFlakeShow : FlakeCommand { addFlag({ .longName = "legacy", - .description = "show the contents of the 'legacyPackages' output", + .description = "Show the contents of the `legacyPackages` output.", .handler = {&showLegacy, true} }); } diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 6fd791f41..79d506ace 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -19,15 +19,15 @@ struct CmdHashBase : Command CmdHashBase(FileIngestionMethod mode) : mode(mode) { - mkFlag(0, "sri", "print hash in SRI format", &base, SRI); - mkFlag(0, "base64", "print hash in base-64", &base, Base64); - mkFlag(0, "base32", "print hash in base-32 (Nix-specific)", &base, Base32); - mkFlag(0, "base16", "print hash in base-16", &base, Base16); + mkFlag(0, "sri", "Print the hash in SRI format.", &base, SRI); + mkFlag(0, "base64", "Print the hash in base-64 format.", &base, Base64); + mkFlag(0, "base32", "Print the hash in base-32 (Nix-specific) format.", &base, Base32); + mkFlag(0, "base16", "Print the hash in base-16 format.", &base, Base16); addFlag(Flag::mkHashTypeFlag("type", &ht)); #if 0 mkFlag() .longName("modulo") - .description("compute hash modulo specified string") + .description("Compute the hash modulo specified the string.") .labels({"modulus"}) .dest(&modulus); #endif @@ -40,15 +40,14 @@ struct CmdHashBase : Command std::string description() override { - const char* d; switch (mode) { case FileIngestionMethod::Flat: - d = "print cryptographic hash of a regular file"; - break; + return "print cryptographic hash of a regular file"; case FileIngestionMethod::Recursive: - d = "print cryptographic hash of the NAR serialisation of a path"; + return "print cryptographic hash of the NAR serialisation of a path"; + default: + assert(false); }; - return d; } void run() override diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 3506c3fcc..50e3b29c4 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -60,37 +60,37 @@ MixFlakeOptions::MixFlakeOptions() { addFlag({ .longName = "recreate-lock-file", - .description = "recreate lock file from scratch", + .description = "Recreate the flake's lock file from scratch.", .handler = {&lockFlags.recreateLockFile, true} }); addFlag({ .longName = "no-update-lock-file", - .description = "do not allow any updates to the lock file", + .description = "Do not allow any updates to the flake's lock file.", .handler = {&lockFlags.updateLockFile, false} }); addFlag({ .longName = "no-write-lock-file", - .description = "do not write the newly generated lock file", + .description = "Do not write the flake's newly generated lock file.", .handler = {&lockFlags.writeLockFile, false} }); addFlag({ .longName = "no-registries", - .description = "don't use flake registries", + .description = "Don't allow lookups in the flake registries.", .handler = {&lockFlags.useRegistries, false} }); addFlag({ .longName = "commit-lock-file", - .description = "commit changes to the lock file", + .description = "Commit changes to the flake's lock file.", .handler = {&lockFlags.commitLockFile, true} }); addFlag({ .longName = "update-input", - .description = "update a specific flake input", + .description = "Update a specific flake input (ignoring its previous entry in the lock file).", .labels = {"input-path"}, .handler = {[&](std::string s) { lockFlags.inputUpdates.insert(flake::parseInputPath(s)); @@ -103,7 +103,7 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "override-input", - .description = "override a specific flake input (e.g. `dwarffs/nixpkgs`)", + .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`).", .labels = {"input-path", "flake-url"}, .handler = {[&](std::string inputPath, std::string flakeRef) { lockFlags.inputOverrides.insert_or_assign( @@ -114,7 +114,7 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "inputs-from", - .description = "use the inputs of the specified flake as registry entries", + .description = "Use the inputs of the specified flake as registry entries.", .labels = {"flake-url"}, .handler = {[&](std::string flakeRef) { auto evalState = getEvalState(); @@ -143,22 +143,22 @@ SourceExprCommand::SourceExprCommand() addFlag({ .longName = "file", .shortName = 'f', - .description = "evaluate *file* rather than the default", + .description = "Interpret installables as attribute paths relative to the Nix expression stored in *file*.", .labels = {"file"}, .handler = {&file}, .completer = completePath }); addFlag({ - .longName ="expr", - .description = "evaluate attributes from *expr*", + .longName = "expr", + .description = "Interpret installables as attribute paths relative to the Nix expression *expr*.", .labels = {"expr"}, .handler = {&expr} }); addFlag({ - .longName ="derivation", - .description = "operate on the store derivation rather than its outputs", + .longName = "derivation", + .description = "Operate on the store derivation rather than its outputs.", .handler = {&operateOn, OperateOn::Derivation}, }); } diff --git a/src/nix/ls.cc b/src/nix/ls.cc index d48287f27..c0b1ecb32 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -17,9 +17,9 @@ struct MixLs : virtual Args, MixJSON MixLs() { - mkFlag('R', "recursive", "list subdirectories recursively", &recursive); - mkFlag('l', "long", "show more file information", &verbose); - mkFlag('d', "directory", "show directories rather than their contents", &showDirectory); + mkFlag('R', "recursive", "List subdirectories recursively.", &recursive); + mkFlag('l', "long", "Show detailed file information.", &verbose); + mkFlag('d', "directory", "Show directories rather than their contents.", &showDirectory); } void listText(ref accessor) diff --git a/src/nix/main.cc b/src/nix/main.cc index b2406fafe..803453dd5 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -69,15 +69,15 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs addFlag({ .longName = "help", - .description = "show usage information", + .description = "Show usage information.", .handler = {[&]() { if (!completions) showHelpAndExit(); }}, }); addFlag({ .longName = "help-config", - .description = "show configuration options", + .description = "Show configuration settings.", .handler = {[&]() { - std::cout << "The following configuration options are available:\n\n"; + std::cout << "The following configuration settings are available:\n\n"; Table2 tbl; std::map settings; globalConfig.getSettings(settings); @@ -91,25 +91,25 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs addFlag({ .longName = "print-build-logs", .shortName = 'L', - .description = "print full build logs on stderr", + .description = "Print full build logs on standard error.", .handler = {[&]() {setLogFormat(LogFormat::barWithLogs); }}, }); addFlag({ .longName = "version", - .description = "show version information", + .description = "Show version information.", .handler = {[&]() { if (!completions) printVersion(programName); }}, }); addFlag({ .longName = "no-net", - .description = "disable substituters and consider all previously downloaded files up-to-date", + .description = "Disable substituters and consider all previously downloaded files up-to-date.", .handler = {[&]() { useNet = false; }}, }); addFlag({ .longName = "refresh", - .description = "consider all previously downloaded files out-of-date", + .description = "Consider all previously downloaded files out-of-date.", .handler = {[&]() { refresh = true; }}, }); } diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 30b6a50f8..0fa88f1bf 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -18,10 +18,10 @@ struct CmdPathInfo : StorePathsCommand, MixJSON CmdPathInfo() { - mkFlag('s', "size", "print size of the NAR dump of each path", &showSize); - mkFlag('S', "closure-size", "print sum size of the NAR dumps of the closure of each path", &showClosureSize); - mkFlag('h', "human-readable", "with -s and -S, print sizes like 1K 234M 5.67G etc.", &humanReadable); - mkFlag(0, "sigs", "show signatures", &showSigs); + mkFlag('s', "size", "Print the size of the NAR serialisation of each path.", &showSize); + mkFlag('S', "closure-size", "Print the sum of the sizes of the NAR serialisations of the closure of each path.", &showClosureSize); + mkFlag('h', "human-readable", "With `-s` and `-S`, print sizes in a human-friendly format such as `5.67G`.", &humanReadable); + mkFlag(0, "sigs", "Show signatures.", &showSigs); } std::string description() override diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index ce8c85ecf..a831dcd15 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -258,14 +258,14 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON { addFlag({ .longName = "name", - .description = "store path name", + .description = "Override the name component of the resulting store path. It defaults to the base name of *url*.", .labels = {"name"}, .handler = {&name} }); addFlag({ .longName = "expected-hash", - .description = "expected hash of the file", + .description = "The expected hash of the file.", .labels = {"hash"}, .handler = {[&](std::string s) { expectedHash = Hash::parseAny(s, hashType); @@ -276,7 +276,9 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON addFlag({ .longName = "executable", - .description = "make the resulting file executable", + .description = + "Make the resulting file executable. Note that this causes the " + "resulting hash to be a NAR hash rather than a flat file hash.", .handler = {&executable, true}, }); diff --git a/src/nix/run.cc b/src/nix/run.cc index 1340dd46f..ec9388234 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -72,7 +72,7 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment addFlag({ .longName = "command", .shortName = 'c', - .description = "command and arguments to be executed; defaults to '$SHELL'", + .description = "Command and arguments to be executed, defaulting to `$SHELL`", .labels = {"command", "args"}, .handler = {[&](std::vector ss) { if (ss.empty()) throw UsageError("--command requires at least one argument"); diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 13f2c8e69..2588a011d 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -19,7 +19,7 @@ struct CmdShowDerivation : InstallablesCommand addFlag({ .longName = "recursive", .shortName = 'r', - .description = "include the dependencies of the specified derivations", + .description = "Include the dependencies of the specified derivations.", .handler = {&recursive, true} }); } diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 14e2c9761..4b6ead6c7 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -16,7 +16,7 @@ struct CmdCopySigs : StorePathsCommand addFlag({ .longName = "substituter", .shortName = 's', - .description = "use signatures from specified store", + .description = "Use signatures from specified store.", .labels = {"store-uri"}, .handler = {[&](std::string s) { substituterUris.push_back(s); }}, }); @@ -101,7 +101,7 @@ struct CmdSignPaths : StorePathsCommand addFlag({ .longName = "key-file", .shortName = 'k', - .description = "file containing the secret signing key", + .description = "File containing the secret signing key.", .labels = {"file"}, .handler = {&secretKeyFile}, .completer = completePath @@ -150,7 +150,7 @@ struct CmdKeyGenerateSecret : Command { addFlag({ .longName = "key-name", - .description = "identifier of the key (e.g. `cache.example.org-1`)", + .description = "Identifier of the key (e.g. `cache.example.org-1`).", .labels = {"name"}, .handler = {&keyName}, }); diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index f3677763c..9c8fef191 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -13,7 +13,7 @@ struct CmdStoreDelete : StorePathsCommand { addFlag({ .longName = "ignore-liveness", - .description = "do not check whether the paths are reachable from a root", + .description = "Do not check whether the paths are reachable from a root.", .handler = {&options.ignoreLiveness, true} }); } diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc index 6e9607d03..a2d74066e 100644 --- a/src/nix/store-gc.cc +++ b/src/nix/store-gc.cc @@ -13,7 +13,7 @@ struct CmdStoreGC : StoreCommand, MixDryRun { addFlag({ .longName = "max", - .description = "stop after freeing `n` bytes of disk space", + .description = "Stop after freeing *n* bytes of disk space.", .labels = {"n"}, .handler = {&options.maxFreed} }); diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 79be31e73..299ea40aa 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -19,14 +19,14 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand addFlag({ .longName = "profile", .shortName = 'p', - .description = "the Nix profile to upgrade", + .description = "The path to the Nix profile to upgrade.", .labels = {"profile-dir"}, .handler = {&profileDir} }); addFlag({ .longName = "nix-store-paths-url", - .description = "URL of the file that contains the store paths of the latest Nix release", + .description = "The URL of the file that contains the store paths of the latest Nix release.", .labels = {"url"}, .handler = {&storePathsUrl} }); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 620109aac..b2963cf74 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -18,13 +18,13 @@ struct CmdVerify : StorePathsCommand CmdVerify() { - mkFlag(0, "no-contents", "do not verify the contents of each store path", &noContents); - mkFlag(0, "no-trust", "do not verify whether each store path is trusted", &noTrust); + mkFlag(0, "no-contents", "Do not verify the contents of each store path.", &noContents); + mkFlag(0, "no-trust", "Do not verify whether each store path is trusted.", &noTrust); addFlag({ .longName = "substituter", .shortName = 's', - .description = "use signatures from specified store", + .description = "Use signatures from the specified store.", .labels = {"store-uri"}, .handler = {[&](std::string s) { substituterUris.push_back(s); }} }); @@ -32,7 +32,7 @@ struct CmdVerify : StorePathsCommand addFlag({ .longName = "sigs-needed", .shortName = 'n', - .description = "require that each path has at least N valid signatures", + .description = "Require that each path has at least *n* valid signatures.", .labels = {"n"}, .handler = {&sigsNeeded} }); diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 297b638cc..7a4ca5172 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -40,7 +40,7 @@ struct CmdWhyDepends : SourceExprCommand addFlag({ .longName = "all", .shortName = 'a', - .description = "show all edges in the dependency graph leading from 'package' to 'dependency', rather than just a shortest path", + .description = "Show all edges in the dependency graph leading from *package* to *dependency*, rather than just a shortest path.", .handler = {&all, true}, }); } From 61216d32e1c0973424d549c9f3065426b51015c9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Jan 2021 23:27:39 +0100 Subject: [PATCH 38/41] Add 'nix store repair' command --- src/libstore/local-store.hh | 4 +--- src/libstore/store-api.hh | 5 +++++ src/nix-store/nix-store.cc | 2 +- src/nix/store-delete.cc | 1 - src/nix/store-repair.cc | 27 +++++++++++++++++++++++++++ src/nix/store-repair.md | 32 ++++++++++++++++++++++++++++++++ 6 files changed, 66 insertions(+), 5 deletions(-) create mode 100644 src/nix/store-repair.cc create mode 100644 src/nix/store-repair.md diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 6d29c5960..6c7ebac1e 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -198,9 +198,7 @@ public: void vacuumDB(); - /* Repair the contents of the given path by redownloading it using - a substituter (if available). */ - void repairPath(const StorePath & path); + void repairPath(const StorePath & path) override; void addSignatures(const StorePath & storePath, const StringSet & sigs) override; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 9bcff08eb..d1b83933a 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -604,6 +604,11 @@ public: virtual ref getFSAccessor() { unsupported("getFSAccessor"); } + /* Repair the contents of the given path by redownloading it using + a substituter (if available). */ + virtual void repairPath(const StorePath & path) + { unsupported("repairPath"); } + /* Add signatures to the specified store path. The signatures are not verified. */ virtual void addSignatures(const StorePath & storePath, const StringSet & sigs) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index e43788bc3..b97f684a4 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -757,7 +757,7 @@ static void opRepairPath(Strings opFlags, Strings opArgs) throw UsageError("no flags expected"); for (auto & i : opArgs) - ensureLocalStore()->repairPath(store->followLinksToStorePath(i)); + store->repairPath(store->followLinksToStorePath(i)); } /* Optimise the disk space usage of the Nix store by hard-linking diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index 9c8fef191..10245978e 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -32,7 +32,6 @@ struct CmdStoreDelete : StorePathsCommand void run(ref store, std::vector storePaths) override { - for (auto & path : storePaths) options.pathsToDelete.insert(path); diff --git a/src/nix/store-repair.cc b/src/nix/store-repair.cc new file mode 100644 index 000000000..1c7a4392e --- /dev/null +++ b/src/nix/store-repair.cc @@ -0,0 +1,27 @@ +#include "command.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdStoreRepair : StorePathsCommand +{ + std::string description() override + { + return "repair store paths"; + } + + std::string doc() override + { + return + #include "store-repair.md" + ; + } + + void run(ref store, std::vector storePaths) override + { + for (auto & path : storePaths) + store->repairPath(path); + } +}; + +static auto rStoreRepair = registerCommand2({"store", "repair"}); diff --git a/src/nix/store-repair.md b/src/nix/store-repair.md new file mode 100644 index 000000000..92d2205a9 --- /dev/null +++ b/src/nix/store-repair.md @@ -0,0 +1,32 @@ +R""( + +# Examples + +* Repair a store path, after determining that it is corrupt: + + ```console + # nix store verify /nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10 + path '/nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10' was + modified! expected hash + 'sha256:1hd5vnh6xjk388gdk841vflicy8qv7qzj2hb7xlyh8lpb43j921l', got + 'sha256:1a25lf78x5wi6pfkrxalf0n13kdaca0bqmjqnp7wfjza2qz5ssgl' + + # nix store repair /nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10 + ``` + +# Description + +This command attempts to "repair" the store paths specified by +*installables* by redownloading them using the available +substituters. If no substitutes are available, then repair is not +possible. + +> **Warning** +> +> During repair, there is a very small time window during which the old +> path (if it exists) is moved out of the way and replaced with the new +> path. If repair is interrupted in between, then the system may be left +> in a broken state (e.g., if the path contains a critical system +> component like the GNU C Library). + +)"" From d33eca8539d2e66759f7b52fa7b0db4a6a1ba673 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Jan 2021 23:31:18 +0100 Subject: [PATCH 39/41] Rename 'nix store sign-paths' to 'nix store sign' --- doc/manual/src/advanced-topics/post-build-hook.md | 4 ++-- src/nix/key-generate-secret.md | 2 +- src/nix/main.cc | 2 +- src/nix/sigs.cc | 6 +++--- tests/signing.sh | 8 ++++---- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/manual/src/advanced-topics/post-build-hook.md b/doc/manual/src/advanced-topics/post-build-hook.md index bbdabed41..fcb52d878 100644 --- a/doc/manual/src/advanced-topics/post-build-hook.md +++ b/doc/manual/src/advanced-topics/post-build-hook.md @@ -53,7 +53,7 @@ set -f # disable globbing export IFS=' ' echo "Signing paths" $OUT_PATHS -nix sign-paths --key-file /etc/nix/key.private $OUT_PATHS +nix store sign --key-file /etc/nix/key.private $OUT_PATHS echo "Uploading paths" $OUT_PATHS exec nix copy --to 's3://example-nix-cache' $OUT_PATHS ``` @@ -63,7 +63,7 @@ exec nix copy --to 's3://example-nix-cache' $OUT_PATHS > The `$OUT_PATHS` variable is a space-separated list of Nix store > paths. In this case, we expect and want the shell to perform word > splitting to make each output path its own argument to `nix -> sign-paths`. Nix guarantees the paths will not contain any spaces, +> store sign`. Nix guarantees the paths will not contain any spaces, > however a store path might contain glob characters. The `set -f` > disables globbing in the shell. diff --git a/src/nix/key-generate-secret.md b/src/nix/key-generate-secret.md index 6ff1e1c9b..4938f637c 100644 --- a/src/nix/key-generate-secret.md +++ b/src/nix/key-generate-secret.md @@ -12,7 +12,7 @@ R""( ```console # nix build nixpkgs#hello - # nix store sign-paths --key-file ./secret-key --recursive ./result + # nix store sign --key-file ./secret-key --recursive ./result ``` Finally, we can verify the store paths using the corresponding diff --git a/src/nix/main.cc b/src/nix/main.cc index 803453dd5..398526020 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -129,7 +129,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs {"make-content-addressable", {"store", "make-content-addressable"}}, {"optimise-store", {"store", "optimise"}}, {"ping-store", {"store", "ping"}}, - {"sign-paths", {"store", "sign-paths"}}, + {"sign-paths", {"store", "sign"}}, {"to-base16", {"hash", "to-base16"}}, {"to-base32", {"hash", "to-base32"}}, {"to-base64", {"hash", "to-base64"}}, diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 4b6ead6c7..3445182f2 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -92,11 +92,11 @@ struct CmdCopySigs : StorePathsCommand static auto rCmdCopySigs = registerCommand2({"store", "copy-sigs"}); -struct CmdSignPaths : StorePathsCommand +struct CmdSign : StorePathsCommand { Path secretKeyFile; - CmdSignPaths() + CmdSign() { addFlag({ .longName = "key-file", @@ -140,7 +140,7 @@ struct CmdSignPaths : StorePathsCommand } }; -static auto rCmdSignPaths = registerCommand2({"store", "sign-paths"}); +static auto rCmdSign = registerCommand2({"store", "sign"}); struct CmdKeyGenerateSecret : Command { diff --git a/tests/signing.sh b/tests/signing.sh index bd6280cc6..6aafbeb91 100644 --- a/tests/signing.sh +++ b/tests/signing.sh @@ -47,8 +47,8 @@ expect 2 nix store verify -r $outPath2 --sigs-needed 1 expect 2 nix store verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1 -# Test "nix store sign-paths". -nix store sign-paths --key-file $TEST_ROOT/sk1 $outPath2 +# Test "nix store sign". +nix store sign --key-file $TEST_ROOT/sk1 $outPath2 nix store verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1 @@ -63,7 +63,7 @@ nix store verify $outPathCA nix store verify $outPathCA --sigs-needed 1000 # Check that signing a content-addressed path doesn't overflow validSigs -nix store sign-paths --key-file $TEST_ROOT/sk1 $outPathCA +nix store sign --key-file $TEST_ROOT/sk1 $outPathCA nix store verify -r $outPathCA --sigs-needed 1000 --trusted-public-keys $pk1 # Copy to a binary cache. @@ -76,7 +76,7 @@ info=$(nix path-info --store file://$cacheDir --json $outPath2) (! [[ $info =~ 'cache2.example.org' ]]) # Verify that adding a signature to a path in a binary cache works. -nix store sign-paths --store file://$cacheDir --key-file $TEST_ROOT/sk2 $outPath2 +nix store sign --store file://$cacheDir --key-file $TEST_ROOT/sk2 $outPath2 info=$(nix path-info --store file://$cacheDir --json $outPath2) [[ $info =~ 'cache1.example.org' ]] [[ $info =~ 'cache2.example.org' ]] From 28ef6ebf914792f9e543e9778248f06e716a859d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Jan 2021 23:51:27 +0100 Subject: [PATCH 40/41] Typo --- src/nix-daemon/nix-daemon.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index fc6195cf0..9227369b8 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -318,7 +318,7 @@ static int main_nix_daemon(int argc, char * * argv) FdSource from(STDIN_FILENO); FdSink to(STDOUT_FILENO); /* Auth hook is empty because in this mode we blindly trust the - standard streams. Limitting access to thoses is explicitly + standard streams. Limiting access to those is explicitly not `nix-daemon`'s responsibility. */ processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){}); } From 7a472a76d4dcbbd0eb7832c0bdcb120d32881e8b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 14 Jan 2021 00:05:04 +0100 Subject: [PATCH 41/41] Add 'nix daemon' command --- src/nix/command.hh | 2 + .../nix-daemon.cc => nix/daemon.cc} | 114 +++++++++++------- src/nix/daemon.md | 21 ++++ src/nix/main.cc | 3 + tests/common.sh.in | 2 +- 5 files changed, 98 insertions(+), 44 deletions(-) rename src/{nix-daemon/nix-daemon.cc => nix/daemon.cc} (77%) create mode 100644 src/nix/daemon.md diff --git a/src/nix/command.hh b/src/nix/command.hh index 3aae57edd..f325cd906 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -13,6 +13,8 @@ namespace nix { extern std::string programPath; +extern char * * savedArgv; + class EvalState; struct Pos; class Store; diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix/daemon.cc similarity index 77% rename from src/nix-daemon/nix-daemon.cc rename to src/nix/daemon.cc index 9227369b8..204d4ce6b 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix/daemon.cc @@ -1,3 +1,4 @@ +#include "command.hh" #include "shared.hh" #include "local-store.hh" #include "remote-store.hh" @@ -150,7 +151,7 @@ static ref openUncachedStore() } -static void daemonLoop(char * * argv) +static void daemonLoop() { if (chdir("/") == -1) throw SysError("cannot change current directory"); @@ -232,9 +233,9 @@ static void daemonLoop(char * * argv) setSigChldAction(false); // For debugging, stuff the pid into argv[1]. - if (peer.pidKnown && argv[1]) { + if (peer.pidKnown && savedArgv[1]) { string processName = std::to_string(peer.pid); - strncpy(argv[1], processName.c_str(), strlen(argv[1])); + strncpy(savedArgv[1], processName.c_str(), strlen(savedArgv[1])); } // Handle the connection. @@ -264,6 +265,48 @@ static void daemonLoop(char * * argv) } } +static void runDaemon(bool stdio) +{ + if (stdio) { + if (auto store = openUncachedStore().dynamic_pointer_cast()) { + auto conn = store->openConnectionWrapper(); + int from = conn->from.fd; + int to = conn->to.fd; + + auto nfds = std::max(from, STDIN_FILENO) + 1; + while (true) { + fd_set fds; + FD_ZERO(&fds); + FD_SET(from, &fds); + FD_SET(STDIN_FILENO, &fds); + if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) + throw SysError("waiting for data from client or server"); + if (FD_ISSET(from, &fds)) { + auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (res == -1) + throw SysError("splicing data from daemon socket to stdout"); + else if (res == 0) + throw EndOfFile("unexpected EOF from daemon socket"); + } + if (FD_ISSET(STDIN_FILENO, &fds)) { + auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (res == -1) + throw SysError("splicing data from stdin to daemon socket"); + else if (res == 0) + return; + } + } + } else { + FdSource from(STDIN_FILENO); + FdSink to(STDOUT_FILENO); + /* Auth hook is empty because in this mode we blindly trust the + standard streams. Limiting access to those is explicitly + not `nix-daemon`'s responsibility. */ + processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){}); + } + } else + daemonLoop(); +} static int main_nix_daemon(int argc, char * * argv) { @@ -285,49 +328,34 @@ static int main_nix_daemon(int argc, char * * argv) initPlugins(); - if (stdio) { - if (auto store = openUncachedStore().dynamic_pointer_cast()) { - auto conn = store->openConnectionWrapper(); - int from = conn->from.fd; - int to = conn->to.fd; - - auto nfds = std::max(from, STDIN_FILENO) + 1; - while (true) { - fd_set fds; - FD_ZERO(&fds); - FD_SET(from, &fds); - FD_SET(STDIN_FILENO, &fds); - if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) - throw SysError("waiting for data from client or server"); - if (FD_ISSET(from, &fds)) { - auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); - if (res == -1) - throw SysError("splicing data from daemon socket to stdout"); - else if (res == 0) - throw EndOfFile("unexpected EOF from daemon socket"); - } - if (FD_ISSET(STDIN_FILENO, &fds)) { - auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE); - if (res == -1) - throw SysError("splicing data from stdin to daemon socket"); - else if (res == 0) - return 0; - } - } - } else { - FdSource from(STDIN_FILENO); - FdSink to(STDOUT_FILENO); - /* Auth hook is empty because in this mode we blindly trust the - standard streams. Limiting access to those is explicitly - not `nix-daemon`'s responsibility. */ - processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){}); - } - } else { - daemonLoop(argv); - } + runDaemon(stdio); return 0; } } static RegisterLegacyCommand r_nix_daemon("nix-daemon", main_nix_daemon); + +struct CmdDaemon : StoreCommand +{ + std::string description() override + { + return "daemon to perform store operations on behalf of non-root clients"; + } + + Category category() override { return catUtility; } + + std::string doc() override + { + return + #include "daemon.md" + ; + } + + void run(ref store) override + { + runDaemon(false); + } +}; + +static auto rCmdDaemon = registerCommand2({"daemon"}); diff --git a/src/nix/daemon.md b/src/nix/daemon.md new file mode 100644 index 000000000..e97016a94 --- /dev/null +++ b/src/nix/daemon.md @@ -0,0 +1,21 @@ +R""( + +# Example + +* Run the daemon in the foreground: + + ```console + # nix daemon + ``` + +# Description + +This command runs the Nix daemon, which is a required component in +multi-user Nix installations. It performs build actions and other +operations on the Nix store on behalf of non-root users. Usually you +don't run the daemon directly; instead it's managed by a service +management framework such as `systemd`. + +Note that this daemon does not fork into the background. + +)"" diff --git a/src/nix/main.cc b/src/nix/main.cc index 398526020..418396280 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -52,6 +52,7 @@ static bool haveInternet() } std::string programPath; +char * * savedArgv; struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { @@ -232,6 +233,8 @@ static auto rCmdHelp = registerCommand("help"); void mainWrapped(int argc, char * * argv) { + savedArgv = argv; + /* The chroot helper needs to be run before any threads have been started. */ if (argc > 0 && argv[0] == chrootHelperName) { diff --git a/tests/common.sh.in b/tests/common.sh.in index 5489c0c44..e3bcab507 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -73,7 +73,7 @@ startDaemon() { # Start the daemon, wait for the socket to appear. !!! # ‘nix-daemon’ should have an option to fork into the background. rm -f $NIX_STATE_DIR/daemon-socket/socket - nix-daemon & + nix daemon & for ((i = 0; i < 30; i++)); do if [ -e $NIX_DAEMON_SOCKET_PATH ]; then break; fi sleep 1