From 02bff90e7bbc04244b3d566d82c945156cb0885b Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 1 Sep 2022 22:02:38 +0200 Subject: [PATCH 01/37] nix flake show: don't evaluate derivations for foreign systems by default --- src/nix/flake.cc | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 3967f1102..5232534c6 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -957,6 +957,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun struct CmdFlakeShow : FlakeCommand, MixJSON { bool showLegacy = false; + bool showForeign = false; CmdFlakeShow() { @@ -965,6 +966,11 @@ struct CmdFlakeShow : FlakeCommand, MixJSON .description = "Show the contents of the `legacyPackages` output.", .handler = {&showLegacy, true} }); + addFlag({ + .longName = "foreign", + .description = "Show the contents of outputs for foreign systems.", + .handler = {&showForeign, true} + }); } std::string description() override @@ -985,6 +991,7 @@ struct CmdFlakeShow : FlakeCommand, MixJSON auto state = getEvalState(); auto flake = std::make_shared(lockFlake()); + auto localSystem = std::string(settings.thisSystem.get()); std::functioncout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--foreign' to show)", headerPrefix)); + else { + logger->warn(fmt("%s omitted (use '--foreign' to show)", concatStringsSep(".", attrPathS))); + } + } else { + if (visitor.isDerivation()) + showDerivation(); + else + throw Error("expected a derivation"); + } } else if (attrPath.size() > 0 && attrPathS[0] == "hydraJobs") { From 6da4cc92d8c546939818b65ba4f1b4ce65d88d6e Mon Sep 17 00:00:00 2001 From: Max Date: Wed, 7 Sep 2022 20:28:30 +0200 Subject: [PATCH 02/37] showForeign -> showAllSystems --- src/nix/flake.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 5232534c6..bb8be872b 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -957,7 +957,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun struct CmdFlakeShow : FlakeCommand, MixJSON { bool showLegacy = false; - bool showForeign = false; + bool showAllSystems = false; CmdFlakeShow() { @@ -967,9 +967,9 @@ struct CmdFlakeShow : FlakeCommand, MixJSON .handler = {&showLegacy, true} }); addFlag({ - .longName = "foreign", - .description = "Show the contents of outputs for foreign systems.", - .handler = {&showForeign, true} + .longName = "all-systems", + .description = "Show the contents of outputs for all systems.", + .handler = {&showAllSystems, true} }); } @@ -1082,11 +1082,11 @@ struct CmdFlakeShow : FlakeCommand, MixJSON || (attrPath.size() == 3 && (attrPathS[0] == "checks" || attrPathS[0] == "packages" || attrPathS[0] == "devShells")) ) { - if (!showForeign && std::string(attrPathS[1]) != localSystem) { + if (!showAllSystems && std::string(attrPathS[1]) != localSystem) { if (!json) - logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--foreign' to show)", headerPrefix)); + logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--all-systems' to show)", headerPrefix)); else { - logger->warn(fmt("%s omitted (use '--foreign' to show)", concatStringsSep(".", attrPathS))); + logger->warn(fmt("%s omitted (use '--all-systems' to show)", concatStringsSep(".", attrPathS))); } } else { if (visitor.isDerivation()) From 15f7fa59bec1dbb4fa94bc1c2f9018b1f14f0d65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Fri, 23 Sep 2022 23:30:29 +0200 Subject: [PATCH 03/37] `unsafeDiscardReferences` Adds a new boolean structured attribute `outputChecks..unsafeDiscardReferences` which disables scanning an output for runtime references. __structuredAttrs = true; outputChecks.out.unsafeDiscardReferences = true; This is useful when creating filesystem images containing their own embedded Nix store: they are self-contained blobs of data with no runtime dependencies. Setting this attribute requires the experimental feature `discard-references` to be enabled. --- .../src/language/advanced-attributes.md | 6 ++--- doc/manual/src/release-notes/rl-next.md | 12 +++++++++ src/libstore/build/local-derivation-goal.cc | 27 ++++++++++++++++--- src/libutil/experimental-features.cc | 1 + src/libutil/experimental-features.hh | 1 + tests/check-refs.nix | 9 +++++++ tests/check-refs.sh | 9 +++++++ 7 files changed, 58 insertions(+), 7 deletions(-) diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 2e7e80ed0..f02425b13 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -207,13 +207,13 @@ Derivations can declare some infrequently used optional attributes. the hash in either hexadecimal or base-32 notation. (See the [`nix-hash` command](../command-ref/nix-hash.md) for information about converting to and from base-32 notation.) - + - [`__contentAddressed`]{#adv-attr-__contentAddressed} If this **experimental** attribute is set to true, then the derivation outputs will be stored in a content-addressed location rather than the traditional input-addressed one. - This only has an effect if the `ca-derivation` experimental feature is enabled. - + This only has an effect if the `ca-derivations` experimental feature is enabled. + Setting this attribute also requires setting `outputHashMode` and `outputHashAlgo` like for *fixed-output derivations* (see above). - [`passAsFile`]{#adv-attr-passAsFile}\ diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 9f491efc8..51320d50a 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -21,3 +21,15 @@ * Error traces have been reworked to provide detailed explanations and more accurate error locations. A short excerpt of the trace is now shown by default when an error occurs. + +* In derivations that use structured attributes, you can now use `unsafeDiscardReferences` + to disable scanning a given output for runtime dependencies: + ```nix + __structuredAttrs = true; + outputChecks.out.unsafeDiscardReferences = true; + ``` + This is useful e.g. when generating self-contained filesystem images with + their own embedded Nix store: hashes found inside such an image refer + to the embedded store and not to the host's Nix store. + + This requires the `discard-references` experimental feature. diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 9d869d513..65340d6ce 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2319,11 +2319,30 @@ DrvOutputs LocalDerivationGoal::registerOutputs() buildUser ? std::optional(buildUser->getUIDRange()) : std::nullopt, inodesSeen); - debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); + bool discardReferences = false; + if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { + if (auto outputChecks = get(*structuredAttrs, "outputChecks")) { + if (auto output = get(*outputChecks, outputName)) { + if (auto unsafeDiscardReferences = get(*output, "unsafeDiscardReferences")) { + settings.requireExperimentalFeature(Xp::DiscardReferences); + if (!unsafeDiscardReferences->is_boolean()) + throw Error("attribute 'outputChecks.\"%s\".unsafeDiscardReferences' of derivation '%s' must be a Boolean", outputName, drvPath.to_string()); + discardReferences = unsafeDiscardReferences->get(); + } + } + } + } - /* Pass blank Sink as we are not ready to hash data at this stage. */ - NullSink blank; - auto references = scanForReferences(blank, actualPath, referenceablePaths); + StorePathSet references; + if (discardReferences) + debug("discarding references of output '%s'", outputName); + else { + debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); + + /* Pass blank Sink as we are not ready to hash data at this stage. */ + NullSink blank; + references = scanForReferences(blank, actualPath, referenceablePaths); + } outputReferencesIfUnregistered.insert_or_assign( outputName, diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index e0902971e..58d762ebb 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -16,6 +16,7 @@ std::map stringifiedXpFeatures = { { Xp::ReplFlake, "repl-flake" }, { Xp::AutoAllocateUids, "auto-allocate-uids" }, { Xp::Cgroups, "cgroups" }, + { Xp::DiscardReferences, "discard-references" }, }; const std::optional parseExperimentalFeature(const std::string_view & name) diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index af775feb0..ac372e03e 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -25,6 +25,7 @@ enum struct ExperimentalFeature ReplFlake, AutoAllocateUids, Cgroups, + DiscardReferences, }; /** diff --git a/tests/check-refs.nix b/tests/check-refs.nix index 9d90b0920..a5e40c9e5 100644 --- a/tests/check-refs.nix +++ b/tests/check-refs.nix @@ -67,4 +67,13 @@ rec { disallowedReferences = [test5]; }; + test11 = makeTest 11 { + __structuredAttrs = true; + outputChecks.out = { + unsafeDiscardReferences = true; + allowedReferences = []; + }; + buildCommand = ''echo ${dep} > "''${outputs[out]}"''; + }; + } diff --git a/tests/check-refs.sh b/tests/check-refs.sh index 16bbabc40..65a72552a 100644 --- a/tests/check-refs.sh +++ b/tests/check-refs.sh @@ -40,3 +40,12 @@ nix-build -o $RESULT check-refs.nix -A test7 # test10 should succeed (no disallowed references). nix-build -o $RESULT check-refs.nix -A test10 + +if isDaemonNewer 2.12pre20230103; then + enableFeatures discard-references + restartDaemon + + # test11 should succeed. + test11=$(nix-build -o $RESULT check-refs.nix -A test11) + [[ -z $(nix-store -q --references "$test11") ]] +fi From 3c968191f1d54a172d2b245d080c67012f766f52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Tue, 3 Jan 2023 18:53:01 +0100 Subject: [PATCH 04/37] move `unsafeDiscardReferences` out of `outputChecks` It's not a check. --- doc/manual/src/release-notes/rl-next.md | 2 +- src/libstore/build/local-derivation-goal.cc | 14 ++++++-------- tests/check-refs.nix | 6 ++---- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 51320d50a..1e8a035b0 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -26,7 +26,7 @@ to disable scanning a given output for runtime dependencies: ```nix __structuredAttrs = true; - outputChecks.out.unsafeDiscardReferences = true; + unsafeDiscardReferences.out = true; ``` This is useful e.g. when generating self-contained filesystem images with their own embedded Nix store: hashes found inside such an image refer diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 65340d6ce..d4ea68f93 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2321,14 +2321,12 @@ DrvOutputs LocalDerivationGoal::registerOutputs() bool discardReferences = false; if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { - if (auto outputChecks = get(*structuredAttrs, "outputChecks")) { - if (auto output = get(*outputChecks, outputName)) { - if (auto unsafeDiscardReferences = get(*output, "unsafeDiscardReferences")) { - settings.requireExperimentalFeature(Xp::DiscardReferences); - if (!unsafeDiscardReferences->is_boolean()) - throw Error("attribute 'outputChecks.\"%s\".unsafeDiscardReferences' of derivation '%s' must be a Boolean", outputName, drvPath.to_string()); - discardReferences = unsafeDiscardReferences->get(); - } + if (auto udr = get(*structuredAttrs, "unsafeDiscardReferences")) { + settings.requireExperimentalFeature(Xp::DiscardReferences); + if (auto output = get(*udr, outputName)) { + if (!output->is_boolean()) + throw Error("attribute 'unsafeDiscardReferences.\"%s\"' of derivation '%s' must be a Boolean", outputName, drvPath.to_string()); + discardReferences = output->get(); } } } diff --git a/tests/check-refs.nix b/tests/check-refs.nix index a5e40c9e5..99d69a226 100644 --- a/tests/check-refs.nix +++ b/tests/check-refs.nix @@ -69,10 +69,8 @@ rec { test11 = makeTest 11 { __structuredAttrs = true; - outputChecks.out = { - unsafeDiscardReferences = true; - allowedReferences = []; - }; + unsafeDiscardReferences.out = true; + outputChecks.out.allowedReferences = []; buildCommand = ''echo ${dep} > "''${outputs[out]}"''; }; From 1fc74afbbae1ae3e78e3c0d2096e81c1e24f9d52 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Thu, 12 Jan 2023 13:42:17 -0800 Subject: [PATCH 05/37] nix/show-config: allow getting the value of a specific setting Instead of needing to run `nix show-config --json | jq -r '."warn-dirty".value'` to view the value of `warn-dirty`, you can now run `nix show-config warn-dirty`. --- src/nix/show-config.cc | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc index 29944e748..3530584f9 100644 --- a/src/nix/show-config.cc +++ b/src/nix/show-config.cc @@ -9,15 +9,44 @@ using namespace nix; struct CmdShowConfig : Command, MixJSON { + std::optional name; + + CmdShowConfig() { + expectArgs({ + .label = {"name"}, + .optional = true, + .handler = {&name}, + }); + } + std::string description() override { - return "show the Nix configuration"; + return "show the Nix configuration or the value of a specific setting"; } Category category() override { return catUtility; } void run() override { + if (name) { + if (json) { + throw UsageError("'--json' is not supported when specifying a setting name"); + } + + std::map settings; + globalConfig.getSettings(settings); + auto setting = settings.find(*name); + + if (setting == settings.end()) { + throw Error("could not find setting '%1%'", *name); + } else { + const auto & value = setting->second.value; + logger->cout("%s", value); + } + + return; + } + if (json) { // FIXME: use appropriate JSON types (bool, ints, etc). logger->cout("%s", globalConfig.toJSON().dump()); From 7f195d058c0077a98c48116c7198d97ab194c74a Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Fri, 13 Jan 2023 07:57:55 -0800 Subject: [PATCH 06/37] tests/config: test retrieving a single setting's value with `nix show-config ` --- tests/config.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/config.sh b/tests/config.sh index 3d0da3cef..723f575ed 100644 --- a/tests/config.sh +++ b/tests/config.sh @@ -51,3 +51,8 @@ exp_features=$(nix show-config | grep '^experimental-features' | cut -d '=' -f 2 [[ $prev != $exp_cores ]] [[ $exp_cores == "4242" ]] [[ $exp_features == "flakes nix-command" ]] + +# Test that it's possible to retrieve a single setting's value +val=$(nix show-config | grep '^warn-dirty' | cut -d '=' -f 2 | xargs) +val2=$(nix show-config warn-dirty) +[[ $val == $val2 ]] From a5919f4754be6f4a9fe091e0ee5538ad85af0050 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Apr 2022 10:26:50 +0200 Subject: [PATCH 07/37] =?UTF-8?q?Move=20the=20default=20profiles=20to=20th?= =?UTF-8?q?e=20user=E2=80=99s=20home?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rather than using `/nix/var/nix/{profiles,gcroots}/per-user/`, put the user profiles and gcroots under `$XDG_DATA_DIR/nix/{profiles,gcroots}`. This means that the daemon no longer needs to manage these paths itself (they are fully handled client-side). In particular, it doesn’t have to `chown` them anymore (removing one need for root). This does change the layout of the gc-roots created by nix-env, and is likely to break some stuff, so I’m not sure how to properly handle that. --- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/local-store.cc | 16 ------------- src/libstore/local-store.hh | 2 -- src/libstore/profiles.cc | 26 ++++++++++++++++----- src/libstore/profiles.hh | 4 ++++ src/libstore/store-api.hh | 3 --- src/libutil/util.cc | 18 ++++++++------ src/libutil/util.hh | 3 +++ src/nix-channel/nix-channel.cc | 4 +++- src/nix/daemon.cc | 1 - tests/remote-store.sh | 4 ---- 11 files changed, 42 insertions(+), 41 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 9ab9b17fe..4fec8fb88 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1517,7 +1517,7 @@ void LocalDerivationGoal::startDaemon() try { daemon::processConnection(store, from, to, daemon::NotTrusted, daemon::Recursive, - [&](Store & store) { store.createUser("nobody", 65535); }); + [&](Store & store) {}); debug("terminated daemon connection"); } catch (SysError &) { ignoreException(); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index be21e3ca0..82edaa9bf 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -201,8 +201,6 @@ LocalStore::LocalStore(const Params & params) throw SysError("could not set permissions on '%s' to 755", perUserDir); } - createUser(getUserName(), getuid()); - /* Optionally, create directories and set permissions for a multi-user install. */ if (getuid() == 0 && settings.buildUsersGroup != "") { @@ -1824,20 +1822,6 @@ void LocalStore::signPathInfo(ValidPathInfo & info) } -void LocalStore::createUser(const std::string & userName, uid_t userId) -{ - for (auto & dir : { - fmt("%s/profiles/per-user/%s", stateDir, userName), - fmt("%s/gcroots/per-user/%s", stateDir, userName) - }) { - createDirs(dir); - if (chmod(dir.c_str(), 0755) == -1) - throw SysError("changing permissions of directory '%s'", dir); - if (chown(dir.c_str(), userId, getgid()) == -1) - throw SysError("changing owner of directory '%s'", dir); - } -} - std::optional> LocalStore::queryRealisationCore_( LocalStore::State & state, const DrvOutput & id) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 06d36a7d5..a84eb7c26 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -281,8 +281,6 @@ private: void signPathInfo(ValidPathInfo & info); void signRealisation(Realisation &); - void createUser(const std::string & userName, uid_t userId) override; - // XXX: Make a generic `Store` method FixedOutputHash hashCAPath( const FileIngestionMethod & method, diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 3e4188188..29ce13b8d 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -280,16 +280,30 @@ std::string optimisticLockProfile(const Path & profile) } +Path profilesDir() +{ + auto profileRoot = getDataDir() + "/nix/profiles"; + createDirs(profileRoot); + return profileRoot; +} + + Path getDefaultProfile() { Path profileLink = getHome() + "/.nix-profile"; try { - if (!pathExists(profileLink)) { - replaceSymlink( - getuid() == 0 - ? settings.nixStateDir + "/profiles/default" - : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()), - profileLink); + // Migrate from the “old-style” profiles stored under `/nix/var`: + // If the link exists and points to the old location, rewrite it to the + // new one (otherwise keep-it as-it-is as it might have been + // intentionnally changed, in which case we shouldn’t touch it) + auto legacyProfile = getuid() == 0 + ? settings.nixStateDir + "/profiles/default" + : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()); + if (!pathExists(profileLink) || + (isLink(profileLink) && + readLink(profileLink) == legacyProfile) + ) { + replaceSymlink(profilesDir() + "/profile", profileLink); } return absPath(readLink(profileLink), dirOf(profileLink)); } catch (Error &) { diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index 408ca039c..73667a798 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -68,6 +68,10 @@ void lockProfile(PathLocks & lock, const Path & profile); rebuilt. */ std::string optimisticLockProfile(const Path & profile); +/* Creates and returns the path to a directory suitable for storing the user’s + profiles. */ +Path profilesDir(); + /* Resolve ~/.nix-profile. If ~/.nix-profile doesn't exist yet, create it. */ Path getDefaultProfile(); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 9eab4b4e5..5807392a7 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -657,9 +657,6 @@ public: return toRealPath(printStorePath(storePath)); } - virtual void createUser(const std::string & userName, uid_t userId) - { } - /* * Synchronises the options of the client with those of the daemon * (a no-op when there’s no daemon) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 993dc1cb6..40faa4bf2 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -537,6 +537,16 @@ std::string getUserName() return name; } +Path getHomeOf(uid_t userId) +{ + std::vector buf(16384); + struct passwd pwbuf; + struct passwd * pw; + if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0 + || !pw || !pw->pw_dir || !pw->pw_dir[0]) + throw Error("cannot determine user's home directory"); + return pw->pw_dir; +} Path getHome() { @@ -558,13 +568,7 @@ Path getHome() } } if (!homeDir) { - std::vector buf(16384); - struct passwd pwbuf; - struct passwd * pw; - if (getpwuid_r(geteuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0 - || !pw || !pw->pw_dir || !pw->pw_dir[0]) - throw Error("cannot determine user's home directory"); - homeDir = pw->pw_dir; + homeDir = getHomeOf(geteuid()); if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 9b149de80..266da0ae3 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -137,6 +137,9 @@ void deletePath(const Path & path, uint64_t & bytesFreed); std::string getUserName(); +/* Return the given user's home directory from /etc/passwd. */ +Path getHomeOf(uid_t userId); + /* Return $HOME or the user's home directory from /etc/passwd. */ Path getHome(); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index cf52b03b4..263d85eea 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -1,9 +1,11 @@ +#include "profiles.hh" #include "shared.hh" #include "globals.hh" #include "filetransfer.hh" #include "store-api.hh" #include "legacy.hh" #include "fetchers.hh" +#include "util.hh" #include #include @@ -166,7 +168,7 @@ static int main_nix_channel(int argc, char ** argv) nixDefExpr = home + "/.nix-defexpr"; // Figure out the name of the channels profile. - profile = fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()); + profile = profilesDir() + "/channels"; enum { cNone, diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index c527fdb0a..19fbbf155 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -248,7 +248,6 @@ static void daemonLoop() querySetting("build-users-group", "") == "") throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!"); #endif - store.createUser(user, peer.uid); }); exit(0); diff --git a/tests/remote-store.sh b/tests/remote-store.sh index 31210ab47..1ae126794 100644 --- a/tests/remote-store.sh +++ b/tests/remote-store.sh @@ -30,7 +30,3 @@ NIX_REMOTE= nix-store --dump-db > $TEST_ROOT/d2 cmp $TEST_ROOT/d1 $TEST_ROOT/d2 killDaemon - -user=$(whoami) -[ -e $NIX_STATE_DIR/gcroots/per-user/$user ] -[ -e $NIX_STATE_DIR/profiles/per-user/$user ] From 0601050755ba39f52fdd2a0fc0478baaf9b1c7b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Apr 2022 10:26:50 +0200 Subject: [PATCH 08/37] Migrate the old profiles to the new location MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make sure that we don’t just create the new profiles directory, but that we also migrate every existing profile to it. --- src/libstore/profiles.cc | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 29ce13b8d..d185a898c 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -293,17 +293,29 @@ Path getDefaultProfile() Path profileLink = getHome() + "/.nix-profile"; try { // Migrate from the “old-style” profiles stored under `/nix/var`: - // If the link exists and points to the old location, rewrite it to the - // new one (otherwise keep-it as-it-is as it might have been - // intentionnally changed, in which case we shouldn’t touch it) + // If the link exists and points to the old location, then: + // - Rewrite it to point to the new location + // - For every generation of the old default profile, create a symlink + // from the new directory to it (so that all the previous profiles + // and generations are still available). auto legacyProfile = getuid() == 0 ? settings.nixStateDir + "/profiles/default" : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()); - if (!pathExists(profileLink) || - (isLink(profileLink) && - readLink(profileLink) == legacyProfile) - ) { - replaceSymlink(profilesDir() + "/profile", profileLink); + auto newProfile = profilesDir() + "/profile"; + if (!pathExists(profileLink) + || (isLink(profileLink) + && readLink(profileLink) == legacyProfile)) { + warn("Migrating the default profile"); + replaceSymlink(newProfile, profileLink); + replaceSymlink(legacyProfile, newProfile); + if (pathExists(legacyProfile)) { + for (auto & oldGen : findGenerations(legacyProfile).first) { + replaceSymlink( + oldGen.path, + dirOf(newProfile) + "/" + + std::string(baseNameOf(oldGen.path))); + } + } } return absPath(readLink(profileLink), dirOf(profileLink)); } catch (Error &) { From 1f02aa4098a61e60062e3ecdb713810e24efa25a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Apr 2022 10:26:50 +0200 Subject: [PATCH 09/37] Test the migration of the user profiles --- tests/common.sh.in | 2 +- tests/local.mk | 1 + tests/user-envs-migration.sh | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 tests/user-envs-migration.sh diff --git a/tests/common.sh.in b/tests/common.sh.in index 73c2d2309..74bbbc8ca 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -62,7 +62,7 @@ readLink() { } clearProfiles() { - profiles="$NIX_STATE_DIR"/profiles + profiles="$HOME"/.local/share/nix/profiles rm -rf $profiles } diff --git a/tests/local.mk b/tests/local.mk index 5ac1ede32..b900e1534 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -17,6 +17,7 @@ nix_tests = \ fetchMercurial.sh \ gc-auto.sh \ user-envs.sh \ + user-envs-migration.sh \ binary-cache.sh \ multiple-outputs.sh \ ca/build.sh \ diff --git a/tests/user-envs-migration.sh b/tests/user-envs-migration.sh new file mode 100644 index 000000000..467c28fbb --- /dev/null +++ b/tests/user-envs-migration.sh @@ -0,0 +1,35 @@ +# Test that the migration of user environments +# (https://github.com/NixOS/nix/pull/5226) does preserve everything + +source common.sh + +if isDaemonNewer "2.4pre20211005"; then + exit 99 +fi + + +killDaemon +unset NIX_REMOTE + +clearStore +clearProfiles +rm -rf ~/.nix-profile + +# Fill the environment using the older Nix +PATH_WITH_NEW_NIX="$PATH" +export PATH="$NIX_DAEMON_PACKAGE/bin:$PATH" + +nix-env -f user-envs.nix -i foo-1.0 +nix-env -f user-envs.nix -i bar-0.1 + +# Migrate to the new profile dir, and ensure that everything’s there +export PATH="$PATH_WITH_NEW_NIX" +nix-env -q # Trigger the migration +( [[ -L ~/.nix-profile ]] && \ + [[ $(readlink ~/.nix-profile) == ~/.local/share/nix/profiles/profile ]] ) || \ + fail "The nix profile should point to the new location" + +(nix-env -q | grep foo && nix-env -q | grep bar && \ + [[ -e ~/.nix-profile/bin/foo ]] && \ + [[ $(nix-env --list-generations | wc -l) == 2 ]]) || + fail "The nix profile should have the same content as before the migration" From c80621dbacc501f8bf43863438a3df087436d13e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 17 Jan 2023 14:15:00 +0100 Subject: [PATCH 10/37] Don't try to migrate existing profiles Doing so would be more dangerous than useful, better leave them as-is if they already exist --- src/libstore/profiles.cc | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index d185a898c..ff62d0972 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -292,30 +292,9 @@ Path getDefaultProfile() { Path profileLink = getHome() + "/.nix-profile"; try { - // Migrate from the “old-style” profiles stored under `/nix/var`: - // If the link exists and points to the old location, then: - // - Rewrite it to point to the new location - // - For every generation of the old default profile, create a symlink - // from the new directory to it (so that all the previous profiles - // and generations are still available). - auto legacyProfile = getuid() == 0 - ? settings.nixStateDir + "/profiles/default" - : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()); - auto newProfile = profilesDir() + "/profile"; - if (!pathExists(profileLink) - || (isLink(profileLink) - && readLink(profileLink) == legacyProfile)) { - warn("Migrating the default profile"); - replaceSymlink(newProfile, profileLink); - replaceSymlink(legacyProfile, newProfile); - if (pathExists(legacyProfile)) { - for (auto & oldGen : findGenerations(legacyProfile).first) { - replaceSymlink( - oldGen.path, - dirOf(newProfile) + "/" - + std::string(baseNameOf(oldGen.path))); - } - } + auto profile = profilesDir() + "/profile"; + if (!pathExists(profileLink)) { + replaceSymlink(profile, profileLink); } return absPath(readLink(profileLink), dirOf(profileLink)); } catch (Error &) { From 6bdf4edb77bea3fdc006e9b8131a43f4dcd0ee42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 17 Jan 2023 14:15:53 +0100 Subject: [PATCH 11/37] Keep the default profile the same MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's used as the “system” profile in a bunch of places, so better not touch it. Besides, it doesn't hurt to keep it since it's owned by root any way, so it doesn't have the `chown` problem that the user profiles had and that led to wanting to move them on the client-side. --- src/libstore/profiles.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index ff62d0972..b202351ce 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -292,7 +292,10 @@ Path getDefaultProfile() { Path profileLink = getHome() + "/.nix-profile"; try { - auto profile = profilesDir() + "/profile"; + auto profile = + getuid() == 0 + ? settings.nixStateDir + "/profiles/default" + : profilesDir() + "/profile"; if (!pathExists(profileLink)) { replaceSymlink(profile, profileLink); } From 4d50995effdaf1d04453293d1afa56e9c8ce6f24 Mon Sep 17 00:00:00 2001 From: Eric Wolf Date: Fri, 20 Jan 2023 10:31:26 +0100 Subject: [PATCH 12/37] Fix url parsing for urls using `file+` `file+https://example.org/test.mp4` should not be rejected with `unexpected authority`. --- src/libutil/tests/url.cc | 21 +++++++++++++++++++++ src/libutil/url.cc | 6 +++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/libutil/tests/url.cc b/src/libutil/tests/url.cc index c3b233797..e0c438b4d 100644 --- a/src/libutil/tests/url.cc +++ b/src/libutil/tests/url.cc @@ -99,6 +99,27 @@ namespace nix { ASSERT_EQ(parsed, expected); } + TEST(parseURL, parsesFilePlusHttpsUrl) { + auto s = "file+https://www.example.org/video.mp4"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "file+https://www.example.org/video.mp4", + .base = "https://www.example.org/video.mp4", + .scheme = "file+https", + .authority = "www.example.org", + .path = "/video.mp4", + .query = (StringMap) { }, + .fragment = "", + }; + + ASSERT_EQ(parsed, expected); + } + + TEST(parseURL, rejectsAuthorityInUrlsWithFileTransportation) { + auto s = "file://www.example.org/video.mp4"; + ASSERT_THROW(parseURL(s), Error); + } TEST(parseURL, parseIPv4Address) { auto s = "http://127.0.0.1:8080/file.tar.gz?download=fast&when=now#hello"; diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 5b7abeb49..4e43455e1 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -30,13 +30,13 @@ ParsedURL parseURL(const std::string & url) auto & query = match[6]; auto & fragment = match[7]; - auto isFile = scheme.find("file") != std::string::npos; + auto transportIsFile = parseUrlScheme(scheme).transport == "file"; - if (authority && *authority != "" && isFile) + if (authority && *authority != "" && transportIsFile) throw BadURL("file:// URL '%s' has unexpected authority '%s'", url, *authority); - if (isFile && path.empty()) + if (transportIsFile && path.empty()) path = "/"; return ParsedURL{ From 74026bb1014460d511534da8a955bee6948716ee Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 20 Jan 2023 14:43:26 +0100 Subject: [PATCH 13/37] tests: Move NixOS tests to tests/nixos This will allow contributors to find them more easily. --- flake.nix | 14 +++++++------- tests/{ => nixos/containers}/containers.nix | 0 tests/{ => nixos/containers}/id-test.nix | 0 tests/{ => nixos/containers}/systemd-nspawn.nix | 0 tests/{ => nixos}/github-flakes.nix | 0 tests/{ => nixos}/nix-copy-closure.nix | 0 tests/{ => nixos}/nss-preload.nix | 0 tests/{ => nixos}/remote-builds.nix | 0 tests/{ => nixos}/setuid.nix | 0 tests/{ => nixos}/sourcehut-flakes.nix | 0 10 files changed, 7 insertions(+), 7 deletions(-) rename tests/{ => nixos/containers}/containers.nix (100%) rename tests/{ => nixos/containers}/id-test.nix (100%) rename tests/{ => nixos/containers}/systemd-nspawn.nix (100%) rename tests/{ => nixos}/github-flakes.nix (100%) rename tests/{ => nixos}/nix-copy-closure.nix (100%) rename tests/{ => nixos}/nss-preload.nix (100%) rename tests/{ => nixos}/remote-builds.nix (100%) rename tests/{ => nixos}/setuid.nix (100%) rename tests/{ => nixos}/sourcehut-flakes.nix (100%) diff --git a/flake.nix b/flake.nix index 573373c42..0dc8c4c2e 100644 --- a/flake.nix +++ b/flake.nix @@ -475,37 +475,37 @@ }; # System tests. - tests.remoteBuilds = import ./tests/remote-builds.nix { + tests.remoteBuilds = import ./tests/nixos/remote-builds.nix { system = "x86_64-linux"; inherit nixpkgs; overlay = self.overlays.default; }; - tests.nix-copy-closure = import ./tests/nix-copy-closure.nix { + tests.nix-copy-closure = import ./tests/nixos/nix-copy-closure.nix { system = "x86_64-linux"; inherit nixpkgs; overlay = self.overlays.default; }; - tests.nssPreload = (import ./tests/nss-preload.nix rec { + tests.nssPreload = (import ./tests/nixos/nss-preload.nix rec { system = "x86_64-linux"; inherit nixpkgs; overlay = self.overlays.default; }); - tests.githubFlakes = (import ./tests/github-flakes.nix rec { + tests.githubFlakes = (import ./tests/nixos/github-flakes.nix rec { system = "x86_64-linux"; inherit nixpkgs; overlay = self.overlays.default; }); - tests.sourcehutFlakes = (import ./tests/sourcehut-flakes.nix rec { + tests.sourcehutFlakes = (import ./tests/nixos/sourcehut-flakes.nix rec { system = "x86_64-linux"; inherit nixpkgs; overlay = self.overlays.default; }); - tests.containers = (import ./tests/containers.nix rec { + tests.containers = (import ./tests/nixos/containers/containers.nix rec { system = "x86_64-linux"; inherit nixpkgs; overlay = self.overlays.default; @@ -514,7 +514,7 @@ tests.setuid = nixpkgs.lib.genAttrs ["i686-linux" "x86_64-linux"] (system: - import ./tests/setuid.nix rec { + import ./tests/nixos/setuid.nix rec { inherit nixpkgs system; overlay = self.overlays.default; }); diff --git a/tests/containers.nix b/tests/nixos/containers/containers.nix similarity index 100% rename from tests/containers.nix rename to tests/nixos/containers/containers.nix diff --git a/tests/id-test.nix b/tests/nixos/containers/id-test.nix similarity index 100% rename from tests/id-test.nix rename to tests/nixos/containers/id-test.nix diff --git a/tests/systemd-nspawn.nix b/tests/nixos/containers/systemd-nspawn.nix similarity index 100% rename from tests/systemd-nspawn.nix rename to tests/nixos/containers/systemd-nspawn.nix diff --git a/tests/github-flakes.nix b/tests/nixos/github-flakes.nix similarity index 100% rename from tests/github-flakes.nix rename to tests/nixos/github-flakes.nix diff --git a/tests/nix-copy-closure.nix b/tests/nixos/nix-copy-closure.nix similarity index 100% rename from tests/nix-copy-closure.nix rename to tests/nixos/nix-copy-closure.nix diff --git a/tests/nss-preload.nix b/tests/nixos/nss-preload.nix similarity index 100% rename from tests/nss-preload.nix rename to tests/nixos/nss-preload.nix diff --git a/tests/remote-builds.nix b/tests/nixos/remote-builds.nix similarity index 100% rename from tests/remote-builds.nix rename to tests/nixos/remote-builds.nix diff --git a/tests/setuid.nix b/tests/nixos/setuid.nix similarity index 100% rename from tests/setuid.nix rename to tests/nixos/setuid.nix diff --git a/tests/sourcehut-flakes.nix b/tests/nixos/sourcehut-flakes.nix similarity index 100% rename from tests/sourcehut-flakes.nix rename to tests/nixos/sourcehut-flakes.nix From 261c25601d9a4efd5245e3ef161fb52bf0543083 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 20 Jan 2023 15:32:31 +0100 Subject: [PATCH 14/37] Use the official, documented NixOS runTest interface --- flake.nix | 55 ++++++++++----------------- tests/nixos/containers/containers.nix | 11 ++---- tests/nixos/github-flakes.nix | 15 ++------ tests/nixos/nix-copy-closure.nix | 17 +++++---- tests/nixos/nss-preload.nix | 16 +++----- tests/nixos/remote-builds.nix | 14 ++----- tests/nixos/setuid.nix | 11 +++--- tests/nixos/sourcehut-flakes.nix | 14 ++----- 8 files changed, 56 insertions(+), 97 deletions(-) diff --git a/flake.nix b/flake.nix index 0dc8c4c2e..31d7f1fcf 100644 --- a/flake.nix +++ b/flake.nix @@ -401,6 +401,18 @@ }; }; + nixos-lib = import (nixpkgs + "/nixos/lib") { }; + + # https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests + runNixOSTestFor = system: test: nixos-lib.runTest { + imports = [ test ]; + hostPkgs = nixpkgsFor.${system}; + defaults = { + nixpkgs.pkgs = nixpkgsFor.${system}; + }; + _module.args.nixpkgs = nixpkgs; + }; + in { # A Nixpkgs overlay that overrides the 'nix' and @@ -475,49 +487,22 @@ }; # System tests. - tests.remoteBuilds = import ./tests/nixos/remote-builds.nix { - system = "x86_64-linux"; - inherit nixpkgs; - overlay = self.overlays.default; - }; + tests.remoteBuilds = runNixOSTestFor "x86_64-linux" ./tests/nixos/remote-builds.nix; - tests.nix-copy-closure = import ./tests/nixos/nix-copy-closure.nix { - system = "x86_64-linux"; - inherit nixpkgs; - overlay = self.overlays.default; - }; + tests.nix-copy-closure = runNixOSTestFor "x86_64-linux" ./tests/nixos/nix-copy-closure.nix; - tests.nssPreload = (import ./tests/nixos/nss-preload.nix rec { - system = "x86_64-linux"; - inherit nixpkgs; - overlay = self.overlays.default; - }); + tests.nssPreload = runNixOSTestFor "x86_64-linux" ./tests/nixos/nss-preload.nix; - tests.githubFlakes = (import ./tests/nixos/github-flakes.nix rec { - system = "x86_64-linux"; - inherit nixpkgs; - overlay = self.overlays.default; - }); + tests.githubFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/github-flakes.nix; - tests.sourcehutFlakes = (import ./tests/nixos/sourcehut-flakes.nix rec { - system = "x86_64-linux"; - inherit nixpkgs; - overlay = self.overlays.default; - }); + tests.sourcehutFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/sourcehut-flakes.nix; - tests.containers = (import ./tests/nixos/containers/containers.nix rec { - system = "x86_64-linux"; - inherit nixpkgs; - overlay = self.overlays.default; - }); + tests.containers = runNixOSTestFor "x86_64-linux" ./tests/nixos/containers/containers.nix; tests.setuid = nixpkgs.lib.genAttrs ["i686-linux" "x86_64-linux"] - (system: - import ./tests/nixos/setuid.nix rec { - inherit nixpkgs system; - overlay = self.overlays.default; - }); + (system: runNixOSTestFor system ./tests/nixos/setuid.nix); + # Make sure that nix-env still produces the exact same result # on a particular version of Nixpkgs. diff --git a/tests/nixos/containers/containers.nix b/tests/nixos/containers/containers.nix index a4856b2df..c8ee78a4a 100644 --- a/tests/nixos/containers/containers.nix +++ b/tests/nixos/containers/containers.nix @@ -1,12 +1,7 @@ # Test whether we can run a NixOS container inside a Nix build using systemd-nspawn. -{ nixpkgs, system, overlay }: +{ lib, nixpkgs, ... }: -with import (nixpkgs + "/nixos/lib/testing-python.nix") { - inherit system; - extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; -}; - -makeTest ({ +{ name = "containers"; nodes = @@ -65,4 +60,4 @@ makeTest ({ host.succeed("[[ $(cat ./result/msg) = 'Hello World' ]]") ''; -}) +} diff --git a/tests/nixos/github-flakes.nix b/tests/nixos/github-flakes.nix index a8b036b17..e4d347691 100644 --- a/tests/nixos/github-flakes.nix +++ b/tests/nixos/github-flakes.nix @@ -1,14 +1,9 @@ -{ nixpkgs, system, overlay }: - -with import (nixpkgs + "/nixos/lib/testing-python.nix") { - inherit system; - extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; -}; - +{ lib, config, nixpkgs, ... }: let + pkgs = config.nodes.client.nixpkgs.pkgs; # Generate a fake root CA and a fake api.github.com / github.com / channels.nixos.org certificate. - cert = pkgs.runCommand "cert" { buildInputs = [ pkgs.openssl ]; } + cert = pkgs.runCommand "cert" { nativeBuildInputs = [ pkgs.openssl ]; } '' mkdir -p $out @@ -92,8 +87,6 @@ let ''; in -makeTest ( - { name = "github-flakes"; @@ -207,4 +200,4 @@ makeTest ( client.succeed("nix build nixpkgs#fuse --tarball-ttl 0") ''; -}) +} diff --git a/tests/nixos/nix-copy-closure.nix b/tests/nixos/nix-copy-closure.nix index 2dc164ae4..66cbfb033 100644 --- a/tests/nixos/nix-copy-closure.nix +++ b/tests/nixos/nix-copy-closure.nix @@ -1,13 +1,16 @@ # Test ‘nix-copy-closure’. -{ nixpkgs, system, overlay }: +{ lib, config, nixpkgs, hostPkgs, ... }: -with import (nixpkgs + "/nixos/lib/testing-python.nix") { - inherit system; - extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; -}; +let + pkgs = config.nodes.client.nixpkgs.pkgs; -makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; pkgD = pkgs.tmux; in { + pkgA = pkgs.cowsay; + pkgB = pkgs.wget; + pkgC = pkgs.hello; + pkgD = pkgs.tmux; + +in { name = "nix-copy-closure"; nodes = @@ -74,4 +77,4 @@ makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; pkgD = pk # ) # client.succeed("nix-store --check-validity ${pkgC}") ''; -}) +} diff --git a/tests/nixos/nss-preload.nix b/tests/nixos/nss-preload.nix index 5a6ff3f68..cef62e95b 100644 --- a/tests/nixos/nss-preload.nix +++ b/tests/nixos/nss-preload.nix @@ -1,11 +1,9 @@ -{ nixpkgs, system, overlay }: - -with import (nixpkgs + "/nixos/lib/testing-python.nix") { - inherit system; - extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; -}; +{ lib, config, nixpkgs, ... }: let + + pkgs = config.nodes.client.nixpkgs.pkgs; + nix-fetch = pkgs.writeText "fetch.nix" '' derivation { # This derivation is an copy from what is available over at @@ -41,9 +39,7 @@ let ''; in -makeTest ( - -rec { +{ name = "nss-preload"; nodes = { @@ -122,4 +118,4 @@ rec { nix-build ${nix-fetch} >&2 """) ''; -}) +} diff --git a/tests/nixos/remote-builds.nix b/tests/nixos/remote-builds.nix index 9f88217fe..696cd2652 100644 --- a/tests/nixos/remote-builds.nix +++ b/tests/nixos/remote-builds.nix @@ -1,15 +1,9 @@ # Test Nix's remote build feature. -{ nixpkgs, system, overlay }: - -with import (nixpkgs + "/nixos/lib/testing-python.nix") { - inherit system; - extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; -}; - -makeTest ( +{ config, lib, hostPkgs, ... }: let + pkgs = config.nodes.client.nixpkgs.pkgs; # The configuration of the remote builders. builder = @@ -75,7 +69,7 @@ in # Create an SSH key on the client. subprocess.run([ - "${pkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" + "${hostPkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" ], capture_output=True, check=True) client.succeed("mkdir -p -m 700 /root/.ssh") client.copy_from_host("key", "/root/.ssh/id_ed25519") @@ -109,4 +103,4 @@ in builder1.block() client.succeed("nix-build ${expr nodes.client.config 4}") ''; -}) +} diff --git a/tests/nixos/setuid.nix b/tests/nixos/setuid.nix index 6784615e4..2b66320dd 100644 --- a/tests/nixos/setuid.nix +++ b/tests/nixos/setuid.nix @@ -1,13 +1,12 @@ # Verify that Linux builds cannot create setuid or setgid binaries. -{ nixpkgs, system, overlay }: +{ lib, config, nixpkgs, ... }: -with import (nixpkgs + "/nixos/lib/testing-python.nix") { - inherit system; - extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; -}; +let + pkgs = config.nodes.machine.nixpkgs.pkgs; -makeTest { +in +{ name = "setuid"; nodes.machine = diff --git a/tests/nixos/sourcehut-flakes.nix b/tests/nixos/sourcehut-flakes.nix index b77496ab6..a76fed020 100644 --- a/tests/nixos/sourcehut-flakes.nix +++ b/tests/nixos/sourcehut-flakes.nix @@ -1,12 +1,8 @@ -{ nixpkgs, system, overlay }: - -with import (nixpkgs + "/nixos/lib/testing-python.nix") -{ - inherit system; - extraConfigurations = [{ nixpkgs.overlays = [ overlay ]; }]; -}; +{ lib, config, hostPkgs, nixpkgs, ... }: let + pkgs = config.nodes.sourcehut.nixpkgs.pkgs; + # Generate a fake root CA and a fake git.sr.ht certificate. cert = pkgs.runCommand "cert" { buildInputs = [ pkgs.openssl ]; } '' @@ -64,8 +60,6 @@ let in -makeTest ( - { name = "sourcehut-flakes"; @@ -164,4 +158,4 @@ makeTest ( client.succeed("nix build nixpkgs#fuse --tarball-ttl 0") ''; - }) +} From 3c08a3e6b68135af126a05e9e3be417b604edded Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 20 Jan 2023 15:35:46 +0100 Subject: [PATCH 15/37] PR template: Specify path to integration tests --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index a268d7caf..344f9405f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -21,7 +21,7 @@ Maintainers: tick if completed or explain if not relevant - [ ] tests, as appropriate - functional tests - `tests/**.sh` - unit tests - `src/*/tests` - - integration tests + - integration tests - `tests/nixos/*` - [ ] documentation in the manual - [ ] code and comments are self-explanatory - [ ] commit message explains why the change was made From a91709a604df24026771550638694075ce34fe45 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 23 Jan 2023 15:30:58 -0500 Subject: [PATCH 16/37] Try to fix #7669 The issue *seems* to be the cross jobs, which are missing the `CXXFLAGS` needed to get rapidcheck. PR #6538 would be really nice to resurrect which will prevent the `configureFlags` from going out of sync between the regular build and the cross build again. --- flake.nix | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 5796c54ea..e3f8c728c 100644 --- a/flake.nix +++ b/flake.nix @@ -653,6 +653,7 @@ inherit system crossSystem; overlays = [ self.overlays.default ]; }; + inherit (nixpkgsCross) lib; in with commonDeps { pkgs = nixpkgsCross; }; nixpkgsCross.stdenv.mkDerivation { name = "nix-${version}"; @@ -665,7 +666,11 @@ nativeBuildInputs = nativeBuildDeps; buildInputs = buildDeps ++ propagatedDeps; - configureFlags = [ "--sysconfdir=/etc" "--disable-doc-gen" ]; + configureFlags = [ + "CXXFLAGS=-I${lib.getDev nixpkgsCross.rapidcheck}/extras/gtest/include" + "--sysconfdir=/etc" + "--disable-doc-gen" + ]; enableParallelBuilding = true; From 57f9dcaeb262f521738c25a05f295b74b910ab9c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Jan 2023 22:00:50 +0000 Subject: [PATCH 17/37] Bump zeebe-io/backport-action from 1.0.1 to 1.1.0 Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 1.0.1 to 1.1.0. - [Release notes](https://github.com/zeebe-io/backport-action/releases) - [Commits](https://github.com/zeebe-io/backport-action/compare/v1.0.1...v1.1.0) --- updated-dependencies: - dependency-name: zeebe-io/backport-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 9f8d14509..ca5af260f 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -21,7 +21,7 @@ jobs: fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v1.0.1 + uses: zeebe-io/backport-action@v1.1.0 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} From dc4aa383e991fbe9fa20b047c6812e4f34a332ea Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Tue, 24 Jan 2023 00:15:12 +0100 Subject: [PATCH 18/37] doc: fix anchor links in and to glossary --- doc/manual/src/command-ref/nix-store.md | 4 ++-- doc/manual/src/glossary.md | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md index f6017481c..403cf285d 100644 --- a/doc/manual/src/command-ref/nix-store.md +++ b/doc/manual/src/command-ref/nix-store.md @@ -82,8 +82,8 @@ paths. Realisation is a somewhat overloaded term: produced through substitutes. If there are no (successful) substitutes, realisation fails. -[valid]: ../glossary.md#validity -[substitutes]: ../glossary.md#substitute +[valid]: ../glossary.md#gloss-validity +[substitutes]: ../glossary.md#gloss-substitute The output path of each derivation is printed on standard output. (For non-derivations argument, the argument itself is printed.) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 5a49af8dc..6004df833 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -156,6 +156,8 @@ to path `Q`, then `Q` is in the closure of `P`. Further, if `Q` references `R` then `R` is also in the closure of `P`. + [closure]: #gloss-closure + - [output path]{#gloss-output-path}\ A [store path] produced by a [derivation]. @@ -172,6 +174,8 @@ - The store path is listed in the Nix database as being valid. - All paths in the store path's [closure] are valid. + [validity]: #gloss-validity + - [user environment]{#gloss-user-env}\ An automatically generated store object that consists of a set of symlinks to “active” applications, i.e., other store paths. These From f58759816de03ac2d233576b8740410a68e9192f Mon Sep 17 00:00:00 2001 From: Andrea Ciceri Date: Wed, 23 Nov 2022 17:25:04 +0100 Subject: [PATCH 19/37] Tighten up the `exportReferencesGraph` tests Add an `$` at the end of the `grep` regex. Without it, `checkRef foo` would always imply `checkRef foo.drv`. We want to tell these situations apart to more precisely test what is going on. --- tests/export-graph.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/export-graph.sh b/tests/export-graph.sh index a1449b34e..4954a6cbc 100644 --- a/tests/export-graph.sh +++ b/tests/export-graph.sh @@ -4,7 +4,7 @@ clearStore clearProfiles checkRef() { - nix-store -q --references $TEST_ROOT/result | grep -q "$1" || fail "missing reference $1" + nix-store -q --references $TEST_ROOT/result | grep -q "$1"'$' || fail "missing reference $1" } # Test the export of the runtime dependency graph. From 0664ba0a67a2e1917ca66fed2174a587d27f3369 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 24 Jan 2023 14:39:45 +0100 Subject: [PATCH 20/37] Revert "fixup: remove boehmgc patch" It is still necessary. Please do your research, or f ask the author, which happens to be me. An evaluator like this is not an environment where "it compiles, so it works" will ever hold. This reverts commit 1c40182b12d5fd462c891b597e1a3f9b912502d5. --- boehmgc-coroutine-sp-fallback.diff | 77 ++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 boehmgc-coroutine-sp-fallback.diff diff --git a/boehmgc-coroutine-sp-fallback.diff b/boehmgc-coroutine-sp-fallback.diff new file mode 100644 index 000000000..8fdafbecb --- /dev/null +++ b/boehmgc-coroutine-sp-fallback.diff @@ -0,0 +1,77 @@ +diff --git a/darwin_stop_world.c b/darwin_stop_world.c +index 3dbaa3fb..36a1d1f7 100644 +--- a/darwin_stop_world.c ++++ b/darwin_stop_world.c +@@ -352,6 +352,7 @@ GC_INNER void GC_push_all_stacks(void) + int nthreads = 0; + word total_size = 0; + mach_msg_type_number_t listcount = (mach_msg_type_number_t)THREAD_TABLE_SZ; ++ size_t stack_limit; + if (!EXPECT(GC_thr_initialized, TRUE)) + GC_thr_init(); + +@@ -407,6 +408,19 @@ GC_INNER void GC_push_all_stacks(void) + GC_push_all_stack_sections(lo, hi, p->traced_stack_sect); + } + if (altstack_lo) { ++ // When a thread goes into a coroutine, we lose its original sp until ++ // control flow returns to the thread. ++ // While in the coroutine, the sp points outside the thread stack, ++ // so we can detect this and push the entire thread stack instead, ++ // as an approximation. ++ // We assume that the coroutine has similarly added its entire stack. ++ // This could be made accurate by cooperating with the application ++ // via new functions and/or callbacks. ++ stack_limit = pthread_get_stacksize_np(p->id); ++ if (altstack_lo >= altstack_hi || altstack_lo < altstack_hi - stack_limit) { // sp outside stack ++ altstack_lo = altstack_hi - stack_limit; ++ } ++ + total_size += altstack_hi - altstack_lo; + GC_push_all_stack(altstack_lo, altstack_hi); + } +diff --git a/pthread_stop_world.c b/pthread_stop_world.c +index 4b2c429..1fb4c52 100644 +--- a/pthread_stop_world.c ++++ b/pthread_stop_world.c +@@ -673,6 +673,8 @@ GC_INNER void GC_push_all_stacks(void) + struct GC_traced_stack_sect_s *traced_stack_sect; + pthread_t self = pthread_self(); + word total_size = 0; ++ size_t stack_limit; ++ pthread_attr_t pattr; + + if (!EXPECT(GC_thr_initialized, TRUE)) + GC_thr_init(); +@@ -722,6 +724,31 @@ GC_INNER void GC_push_all_stacks(void) + hi = p->altstack + p->altstack_size; + /* FIXME: Need to scan the normal stack too, but how ? */ + /* FIXME: Assume stack grows down */ ++ } else { ++ if (pthread_getattr_np(p->id, &pattr)) { ++ ABORT("GC_push_all_stacks: pthread_getattr_np failed!"); ++ } ++ if (pthread_attr_getstacksize(&pattr, &stack_limit)) { ++ ABORT("GC_push_all_stacks: pthread_attr_getstacksize failed!"); ++ } ++ if (pthread_attr_destroy(&pattr)) { ++ ABORT("GC_push_all_stacks: pthread_attr_destroy failed!"); ++ } ++ // When a thread goes into a coroutine, we lose its original sp until ++ // control flow returns to the thread. ++ // While in the coroutine, the sp points outside the thread stack, ++ // so we can detect this and push the entire thread stack instead, ++ // as an approximation. ++ // We assume that the coroutine has similarly added its entire stack. ++ // This could be made accurate by cooperating with the application ++ // via new functions and/or callbacks. ++ #ifndef STACK_GROWS_UP ++ if (lo >= hi || lo < hi - stack_limit) { // sp outside stack ++ lo = hi - stack_limit; ++ } ++ #else ++ #error "STACK_GROWS_UP not supported in boost_coroutine2 (as of june 2021), so we don't support it in Nix." ++ #endif + } + GC_push_all_stack_sections(lo, hi, traced_stack_sect); + # ifdef STACK_GROWS_UP From 8270dccf60948fe8a6b6817c81d67eee98c07df3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 24 Jan 2023 14:57:18 +0100 Subject: [PATCH 21/37] Actually complete the revert --- flake.nix | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 573373c42..773e693ba 100644 --- a/flake.nix +++ b/flake.nix @@ -128,9 +128,14 @@ }); propagatedDeps = - [ (boehmgc.override { + [ ((boehmgc.override { enableLargeConfig = true; + }).overrideAttrs(o: { + patches = (o.patches or []) ++ [ + ./boehmgc-coroutine-sp-fallback.diff + ]; }) + ) nlohmann_json ]; }; From 46054f932b6a7dceef127389f83bf7438d1731d0 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 24 Jan 2023 15:11:55 +0100 Subject: [PATCH 22/37] Update boehmgc-coroutine-sp-fallback.diff --- boehmgc-coroutine-sp-fallback.diff | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/boehmgc-coroutine-sp-fallback.diff b/boehmgc-coroutine-sp-fallback.diff index 8fdafbecb..2826486fb 100644 --- a/boehmgc-coroutine-sp-fallback.diff +++ b/boehmgc-coroutine-sp-fallback.diff @@ -31,19 +31,19 @@ index 3dbaa3fb..36a1d1f7 100644 GC_push_all_stack(altstack_lo, altstack_hi); } diff --git a/pthread_stop_world.c b/pthread_stop_world.c -index 4b2c429..1fb4c52 100644 +index b5d71e62..aed7b0bf 100644 --- a/pthread_stop_world.c +++ b/pthread_stop_world.c -@@ -673,6 +673,8 @@ GC_INNER void GC_push_all_stacks(void) - struct GC_traced_stack_sect_s *traced_stack_sect; - pthread_t self = pthread_self(); - word total_size = 0; +@@ -768,6 +768,8 @@ STATIC void GC_restart_handler(int sig) + /* world is stopped. Should not fail if it isn't. */ + GC_INNER void GC_push_all_stacks(void) + { + size_t stack_limit; + pthread_attr_t pattr; - - if (!EXPECT(GC_thr_initialized, TRUE)) - GC_thr_init(); -@@ -722,6 +724,31 @@ GC_INNER void GC_push_all_stacks(void) + GC_bool found_me = FALSE; + size_t nthreads = 0; + int i; +@@ -851,6 +853,31 @@ GC_INNER void GC_push_all_stacks(void) hi = p->altstack + p->altstack_size; /* FIXME: Need to scan the normal stack too, but how ? */ /* FIXME: Assume stack grows down */ From 734c5fdcd62ead1b28e7f7fa3962ce0b542d5846 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Tue, 24 Jan 2023 16:37:50 +0100 Subject: [PATCH 23/37] Fix 'destructor called on non-final ...' warning clangStdenv compiles with a single warning: ``` warning: destructor called on non-final 'nix::PosAdapter' that has virtual functions but non-virtual destructor [-Wdelete-non-abstract-non-virtual-dtor] ``` This fixes the warning by making the destructor of PosAdapter virtual, deffering to the correct destructor from the concrete child classes. This has no impact in the end, as none of these classes have specific destructors. Technicaly, it may be faster not to have this indirection, but as per the warning, there is only one place where we have to delete abstract PosAdapter values. Not worth bikesheding I guess. --- src/libutil/error.hh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 7d236028c..0ebeaba61 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -74,6 +74,8 @@ struct AbstractPos virtual void print(std::ostream & out) const = 0; std::optional getCodeLines() const; + + virtual ~AbstractPos() = default; }; std::ostream & operator << (std::ostream & str, const AbstractPos & pos); From 816031173c8624f2c0b4fec6dc97f3b05a7132b7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 24 Jan 2023 18:53:46 -0500 Subject: [PATCH 24/37] Fix the 2.13 changelog It is just the new CLI that gets the `^` syntax. The old CLI already has a (slightly different) `!` syntax. Fixes #7682 --- doc/manual/src/release-notes/rl-2.13.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/release-notes/rl-2.13.md b/doc/manual/src/release-notes/rl-2.13.md index 2ebf19f60..d8db57a48 100644 --- a/doc/manual/src/release-notes/rl-2.13.md +++ b/doc/manual/src/release-notes/rl-2.13.md @@ -25,11 +25,11 @@ * Allow explicitly selecting outputs in a store derivation installable, just like we can do with other sorts of installables. For example, ```shell-session - # nix-build /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^dev + # nix build /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^dev ``` now works just as ```shell-session - # nix-build glibc^dev + # nix build glibc^dev ``` does already. From 75892710f833e4f5fa304eacfd0bfb45634702ec Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 24 Jan 2023 19:19:19 -0500 Subject: [PATCH 25/37] Fix the coverage job See https://hydra.nixos.org/build/206790960 --- flake.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flake.nix b/flake.nix index 62ea3cf00..2e9e73934 100644 --- a/flake.nix +++ b/flake.nix @@ -465,6 +465,10 @@ src = self; + configureFlags = [ + "CXXFLAGS=-I${lib.getDev pkgs.rapidcheck}/extras/gtest/include" + ]; + enableParallelBuilding = true; nativeBuildInputs = nativeBuildDeps; From f465e378c431e9efa84e03a0446a1380ceb18485 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 25 Jan 2023 08:58:41 -0500 Subject: [PATCH 26/37] Update doc/manual/src/release-notes/rl-2.13.md Co-authored-by: Eelco Dolstra --- doc/manual/src/release-notes/rl-2.13.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/release-notes/rl-2.13.md b/doc/manual/src/release-notes/rl-2.13.md index d8db57a48..168708113 100644 --- a/doc/manual/src/release-notes/rl-2.13.md +++ b/doc/manual/src/release-notes/rl-2.13.md @@ -29,7 +29,7 @@ ``` now works just as ```shell-session - # nix build glibc^dev + # nix build nixpkgs#glibc^dev ``` does already. From a96156c58f94165e8dc61981ab9fa6e97a963e96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sol=C3=A8ne=20Rapenne?= Date: Wed, 14 Dec 2022 14:01:29 +0100 Subject: [PATCH 27/37] warnings: enhance the case of untrusted substituter for untrusted user --- src/libstore/daemon.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 12596ba49..3731b21b8 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -222,7 +222,8 @@ struct ClientSettings else if (!hasSuffix(s, "/") && trusted.count(s + "/")) subs.push_back(s + "/"); else - warn("ignoring untrusted substituter '%s'", s); + warn("ignoring untrusted substituter '%s', you are not a trusted user.\n" + "More information about 'trusted-substituters' option in nix.conf man page", s); res = subs; return true; }; From 64951d9125fc223bbeb939b1c774533a8c6ded98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sol=C3=A8ne=20Rapenne?= Date: Tue, 3 Jan 2023 15:35:28 +0100 Subject: [PATCH 28/37] Update src/libstore/daemon.cc Co-authored-by: Valentin Gagarin --- src/libstore/daemon.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 3731b21b8..e2a7dab35 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -223,7 +223,7 @@ struct ClientSettings subs.push_back(s + "/"); else warn("ignoring untrusted substituter '%s', you are not a trusted user.\n" - "More information about 'trusted-substituters' option in nix.conf man page", s); + "Run `man nix.conf` for more information on the `substituters` configuration option.", s); res = subs; return true; }; From 6b2729c81e1e0d37ba3680e36df4769d35d13c12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sol=C3=A8ne=20Rapenne?= Date: Fri, 20 Jan 2023 09:46:28 +0100 Subject: [PATCH 29/37] improve documentation about substituters and trusted users MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt --- src/libstore/globals.hh | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 7111def92..c3ccb5e11 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -570,11 +570,15 @@ public: {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="}, "trusted-public-keys", R"( - A whitespace-separated list of public keys. When paths are copied - from another Nix store (such as a binary cache), they must be - signed with one of these keys. For example: - `cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=`. + A whitespace-separated list of public keys. + + At least one of the following condition must be met + for Nix to accept copying a store object from another + Nix store (such as a substituter): + + - the store object has been signed using a key in the trusted keys list + - the [`require-sigs`](#conf-require-sigs) option has been set to `false` + - the store object is [output-addressed](@docroot@/glossary.md#gloss-output-addressed-store-object) )", {"binary-cache-public-keys"}}; @@ -670,13 +674,14 @@ public: independently. Lower value means higher priority. The default is `https://cache.nixos.org`, with a Priority of 40. - Nix will copy a store path from a remote store only if one - of the following is true: + At least one of the following conditions must be met for Nix to use + a substituter: - - the store object is signed by one of the [`trusted-public-keys`](#conf-trusted-public-keys) - the substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list - - the [`require-sigs`](#conf-require-sigs) option has been set to `false` - - the store object is [output-addressed](@docroot@/glossary.md#gloss-output-addressed-store-object) + - the user calling Nix is in the [`trusted-users`](#conf-trusted-users) list + + In addition, each store path should be trusted as described + in [`trusted-public-keys`](#conf-trusted-public-keys) )", {"binary-caches"}}; From 79c084cb598221f5910c429127e8da33cff6a206 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 27 Jan 2023 10:13:05 +0100 Subject: [PATCH 30/37] Add a test for `nix flake show` --- tests/flakes/common.sh | 4 ++++ tests/flakes/show.sh | 39 +++++++++++++++++++++++++++++++++++++++ tests/local.mk | 1 + 3 files changed, 44 insertions(+) create mode 100644 tests/flakes/show.sh diff --git a/tests/flakes/common.sh b/tests/flakes/common.sh index c333733c2..fdcdd4aef 100644 --- a/tests/flakes/common.sh +++ b/tests/flakes/common.sh @@ -20,6 +20,10 @@ writeSimpleFlake() { foo = import ./simple.nix; default = foo; }; + packages.someOtherSystem = rec { + foo = import ./simple.nix; + default = foo; + }; # To test "nix flake init". legacyPackages.x86_64-linux.hello = import ./simple.nix; diff --git a/tests/flakes/show.sh b/tests/flakes/show.sh new file mode 100644 index 000000000..995de8dc3 --- /dev/null +++ b/tests/flakes/show.sh @@ -0,0 +1,39 @@ +source ./common.sh + +flakeDir=$TEST_ROOT/flake +mkdir -p "$flakeDir" + +writeSimpleFlake "$flakeDir" +cd "$flakeDir" + + +# By default: Only show the packages content for the current system and no +# legacyPackages at all +nix flake show --json > show-output.json +nix eval --impure --expr ' +let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); +in +assert show_output.packages.someOtherSystem.default == {}; +assert show_output.packages.${builtins.currentSystem}.default.name == "simple"; +assert show_output.legacyPackages.${builtins.currentSystem} == {}; +true +' + +# With `--all-systems`, show the packages for all systems +nix flake show --json --all-systems > show-output.json +nix eval --impure --expr ' +let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); +in +assert show_output.packages.someOtherSystem.default.name == "simple"; +assert show_output.legacyPackages.${builtins.currentSystem} == {}; +true +' + +# With `--legacy`, show the legacy packages +nix flake show --json --legacy > show-output.json +nix eval --impure --expr ' +let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); +in +assert show_output.legacyPackages.${builtins.currentSystem}.hello.name == "simple"; +true +' diff --git a/tests/local.mk b/tests/local.mk index 5ac1ede32..c8ee82e1a 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -113,6 +113,7 @@ nix_tests = \ store-ping.sh \ fetchClosure.sh \ completions.sh \ + flakes/show.sh \ impure-derivations.sh \ path-from-hash-part.sh \ toString-path.sh From 1cba5984a68a489c4a56691032e4c87991c678f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Wed, 25 Jan 2023 15:10:35 +0100 Subject: [PATCH 31/37] getDefaultNixPath: actually respect `{restrict,pure}-eval` Previously, getDefaultNixPath was called too early: at initialisation time, before CLI and config have been processed, when `restrictEval` and `pureEval` both have their default value `false`. Call it when initialising the EvalState instead, and use `setDefault`. --- src/libexpr/eval.cc | 36 +++++++++++++++++++++--------------- src/libexpr/eval.hh | 4 ++-- tests/nix_path.sh | 5 +++++ tests/restricted.sh | 3 +++ 4 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 1828b8c2e..a48968656 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -519,6 +519,7 @@ EvalState::EvalState( static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes"); /* Initialise the Nix expression search path. */ + evalSettings.nixPath.setDefault(evalSettings.getDefaultNixPath()); if (!evalSettings.pureEval) { for (auto & i : _searchPath) addToSearchPath(i); for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i); @@ -2472,30 +2473,35 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { EvalSettings::EvalSettings() { - auto var = getEnv("NIX_PATH"); - if (var) nixPath = parseNixPath(*var); } +/* impure => NIX_PATH or a default path + * restrict-eval => NIX_PATH + * pure-eval => empty + */ Strings EvalSettings::getDefaultNixPath() { - Strings res; - auto add = [&](const Path & p, const std::string & s = std::string()) { - if (pathExists(p)) { - if (s.empty()) { - res.push_back(p); - } else { - res.push_back(s + "=" + p); - } - } - }; + if (pureEval) + return {}; + + auto var = getEnv("NIX_PATH"); + if (var) { + return parseNixPath(*var); + } else if (restrictEval) { + return {}; + } else { + Strings res; + auto add = [&](const Path & p, const std::optional & s = std::nullopt) { + if (pathExists(p)) + res.push_back(s ? *s + "=" + p : p); + }; - if (!evalSettings.restrictEval && !evalSettings.pureEval) { add(getHome() + "/.nix-defexpr/channels"); add(settings.nixStateDir + "/profiles/per-user/root/channels/nixpkgs", "nixpkgs"); add(settings.nixStateDir + "/profiles/per-user/root/channels"); - } - return res; + return res; + } } bool EvalSettings::isPseudoUrl(std::string_view s) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index e4d5906bd..876a6ae0e 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -570,7 +570,7 @@ struct EvalSettings : Config { EvalSettings(); - static Strings getDefaultNixPath(); + Strings getDefaultNixPath(); static bool isPseudoUrl(std::string_view s); @@ -580,7 +580,7 @@ struct EvalSettings : Config "Whether builtin functions that allow executing native code should be enabled."}; Setting nixPath{ - this, getDefaultNixPath(), "nix-path", + this, {}, "nix-path", "List of directories to be searched for `<...>` file references."}; Setting restrictEval{ diff --git a/tests/nix_path.sh b/tests/nix_path.sh index 2b222b4a1..d16fb4bb2 100644 --- a/tests/nix_path.sh +++ b/tests/nix_path.sh @@ -12,3 +12,8 @@ nix-instantiate --eval -E '' --restrict-eval [[ $(nix-instantiate --find-file by-absolute-path/simple.nix) = $PWD/simple.nix ]] [[ $(nix-instantiate --find-file by-relative-path/simple.nix) = $PWD/simple.nix ]] + +unset NIX_PATH + +[[ $(nix-instantiate --option nix-path by-relative-path=. --find-file by-relative-path/simple.nix) = "$PWD/simple.nix" ]] +[[ $(NIX_PATH= nix-instantiate --option nix-path by-relative-path=. --find-file by-relative-path/simple.nix) = "$PWD/simple.nix" ]] diff --git a/tests/restricted.sh b/tests/restricted.sh index 9bd16cf51..3b6ee2af1 100644 --- a/tests/restricted.sh +++ b/tests/restricted.sh @@ -17,6 +17,9 @@ nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/nix-channel' (! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ') nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ' -I src=. +# no default NIX_PATH +(unset NIX_PATH; ! nix-instantiate --restrict-eval --find-file .) + p=$(nix eval --raw --expr "builtins.fetchurl file://$(pwd)/restricted.sh" --impure --restrict-eval --allowed-uris "file://$(pwd)") cmp $p restricted.sh From dba9173a1d8cb1dd40e5922d009cb3a434e081c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Fri, 27 Jan 2023 15:25:07 +0100 Subject: [PATCH 32/37] Document default `nix-path` value --- src/libexpr/eval.hh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 876a6ae0e..2340ef67b 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -581,7 +581,14 @@ struct EvalSettings : Config Setting nixPath{ this, {}, "nix-path", - "List of directories to be searched for `<...>` file references."}; + R"( + List of directories to be searched for `<...>` file references. + + If [pure evaluation](#conf-pure-eval) is disabled, + this is initialised using the [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) + environment variable, or, if it is unset and [restricted evaluation](#conf-restrict-eval) + is disabled, a default search path including the user's and `root`'s channels. + )"}; Setting restrictEval{ this, false, "restrict-eval", From 02e81cdf6276dbaaa5ecb720fb3244d5cf2c5e8e Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 27 Jan 2023 23:59:48 +0100 Subject: [PATCH 33/37] apply showAllSystems to legacyPackages as well --- src/nix/flake.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 2e4d2cf3a..d2f68c37c 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1121,6 +1121,12 @@ struct CmdFlakeShow : FlakeCommand, MixJSON else { logger->warn(fmt("%s omitted (use '--legacy' to show)", concatStringsSep(".", attrPathS))); } + } else if (!showAllSystems && std::string(attrPathS[1]) != localSystem) { + if (!json) + logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--all-systems' to show)", headerPrefix)); + else { + logger->warn(fmt("%s omitted (use '--all-systems' to show)", concatStringsSep(".", attrPathS))); + } } else { if (visitor.isDerivation()) showDerivation(); From 7408776b003d8a7670c5d8eb393f22899a662b25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 30 Jan 2023 10:32:23 +0100 Subject: [PATCH 34/37] Fix the release-notes Slightly butchered during the merge --- doc/manual/src/release-notes/rl-next.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 2f0f89cc6..7e8344e63 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,3 +1,5 @@ +# Release X.Y (202?-??-??) + * A new function `builtins.readFileType` is available. It is similar to `builtins.readDir` but acts on a single file or directory. @@ -6,7 +8,7 @@ discovered by making multiple syscalls. This change makes these operations lazy such that these lookups will only be performed if the attribute is used. This optimization affects a minority of filesystems and operating systems. - + * In derivations that use structured attributes, you can now use `unsafeDiscardReferences` to disable scanning a given output for runtime dependencies: ```nix @@ -17,4 +19,4 @@ their own embedded Nix store: hashes found inside such an image refer to the embedded store and not to the host's Nix store. - This requires the `discard-references` experimental feature. \ No newline at end of file + This requires the `discard-references` experimental feature. From bc6e65e26fd54f9c27a0344b86fc813820a7e9e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 30 Jan 2023 10:44:10 +0100 Subject: [PATCH 35/37] Fix the flakes/show test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Don't hardcode “x86_64-linux” as this won't work too nicely on other platforms --- tests/flakes/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/flakes/common.sh b/tests/flakes/common.sh index fdcdd4aef..9d79080cd 100644 --- a/tests/flakes/common.sh +++ b/tests/flakes/common.sh @@ -26,7 +26,7 @@ writeSimpleFlake() { }; # To test "nix flake init". - legacyPackages.x86_64-linux.hello = import ./simple.nix; + legacyPackages.$system.hello = import ./simple.nix; }; } EOF From 3ac9f1658a482b3bf94b4a78263ea8e1ebea5b13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 30 Jan 2023 11:21:52 +0100 Subject: [PATCH 36/37] Fix the flakes init test Things leading to another... --- tests/flakes/init.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/flakes/init.sh b/tests/flakes/init.sh index 36cb9956a..2d4c77ba1 100644 --- a/tests/flakes/init.sh +++ b/tests/flakes/init.sh @@ -41,8 +41,8 @@ cat > $templatesDir/trivial/flake.nix < Date: Mon, 30 Jan 2023 09:35:25 -0500 Subject: [PATCH 37/37] Avoid some `StorePath` <-> `Path` round trips Avoid needless work and throwing away invariants. These conversions date back to when `StorePath` was in Rust and there were issues with it missing utility methods. --- src/libstore/store-api.cc | 8 ++++---- src/nix/verify.cc | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 5130409d4..31bd9318f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -742,13 +742,13 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m std::condition_variable wakeup; ThreadPool pool; - auto doQuery = [&](const Path & path) { + auto doQuery = [&](const StorePath & path) { checkInterrupt(); - queryPathInfo(parseStorePath(path), {[path, this, &state_, &wakeup](std::future> fut) { + queryPathInfo(path, {[path, this, &state_, &wakeup](std::future> fut) { auto state(state_.lock()); try { auto info = fut.get(); - state->valid.insert(parseStorePath(path)); + state->valid.insert(path); } catch (InvalidPath &) { } catch (...) { state->exc = std::current_exception(); @@ -760,7 +760,7 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m }; for (auto & path : paths) - pool.enqueue(std::bind(doQuery, printStorePath(path))); // FIXME + pool.enqueue(std::bind(doQuery, path)); pool.process(); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index efa2434dc..0b306cc11 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -81,14 +81,14 @@ struct CmdVerify : StorePathsCommand ThreadPool pool; - auto doPath = [&](const Path & storePath) { + auto doPath = [&](const StorePath & storePath) { try { checkInterrupt(); MaintainCount> mcActive(active); update(); - auto info = store->queryPathInfo(store->parseStorePath(storePath)); + auto info = store->queryPathInfo(storePath); // Note: info->path can be different from storePath // for binary cache stores when using --all (since we @@ -173,7 +173,7 @@ struct CmdVerify : StorePathsCommand }; for (auto & storePath : storePaths) - pool.enqueue(std::bind(doPath, store->printStorePath(storePath))); + pool.enqueue(std::bind(doPath, storePath)); pool.process();