diff --git a/misc/bash/completion.sh b/misc/bash/completion.sh
index bc184edd6..bea2a40bc 100644
--- a/misc/bash/completion.sh
+++ b/misc/bash/completion.sh
@@ -4,13 +4,14 @@ function _complete_nix {
_get_comp_words_by_ref -n ':=&' words cword cur
local have_type
while IFS= read -r line; do
+ local completion=${line%% *}
if [[ -z $have_type ]]; then
have_type=1
- if [[ $line = filenames ]]; then
+ if [[ $completion = filenames ]]; then
compopt -o filenames
fi
else
- COMPREPLY+=("$line")
+ COMPREPLY+=("$completion")
fi
done < <(NIX_GET_COMPLETIONS=$cword "${words[@]}")
__ltrim_colon_completions "$cur"
diff --git a/misc/zsh/completion.zsh b/misc/zsh/completion.zsh
new file mode 100644
index 000000000..d4df6447e
--- /dev/null
+++ b/misc/zsh/completion.zsh
@@ -0,0 +1,21 @@
+function _nix() {
+ local ifs_bk="$IFS"
+ local input=("${(Q)words[@]}")
+ IFS=$'\n'
+ local res=($(NIX_GET_COMPLETIONS=$((CURRENT - 1)) "$input[@]"))
+ IFS="$ifs_bk"
+ local tpe="${${res[1]}%%> *}"
+ local -a suggestions
+ declare -a suggestions
+ for suggestion in ${res:1}; do
+ # FIXME: This doesn't work properly if the suggestion word contains a `:`
+ # itself
+ suggestions+="${suggestion/ /:}"
+ done
+ if [[ "$tpe" == filenames ]]; then
+ compadd -f
+ fi
+ _describe 'nix' suggestions
+}
+
+compdef _nix nix
diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc
index 4367ee810..2426882ca 100644
--- a/src/libfetchers/registry.cc
+++ b/src/libfetchers/registry.cc
@@ -3,6 +3,7 @@
#include "util.hh"
#include "globals.hh"
#include "store-api.hh"
+#include "local-fs-store.hh"
#include <nlohmann/json.hpp>
diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc
index 3411e2d7a..9151a0344 100644
--- a/src/libmain/common-args.cc
+++ b/src/libmain/common-args.cc
@@ -44,7 +44,7 @@ MixCommonArgs::MixCommonArgs(const string & programName)
globalConfig.getSettings(settings);
for (auto & s : settings)
if (hasPrefix(s.first, prefix))
- completions->insert(s.first);
+ completions->add(s.first, s.second.description);
}
}
});
diff --git a/src/libstore/build.cc b/src/libstore/build/derivation-goal.cc
similarity index 71%
rename from src/libstore/build.cc
rename to src/libstore/build/derivation-goal.cc
index 05a9ef088..fda05f0e9 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -1,53 +1,36 @@
-#include "references.hh"
-#include "pathlocks.hh"
-#include "globals.hh"
-#include "local-store.hh"
-#include "util.hh"
-#include "archive.hh"
-#include "affinity.hh"
+#include "derivation-goal.hh"
+#include "hook-instance.hh"
+#include "worker.hh"
#include "builtins.hh"
#include "builtins/buildenv.hh"
-#include "filetransfer.hh"
+#include "references.hh"
#include "finally.hh"
-#include "compression.hh"
+#include "util.hh"
+#include "archive.hh"
#include "json.hh"
-#include "nar-info.hh"
-#include "parsed-derivations.hh"
-#include "machines.hh"
+#include "compression.hh"
#include "daemon.hh"
#include "worker-protocol.hh"
#include "topo-sort.hh"
#include "callback.hh"
-#include <algorithm>
-#include <iostream>
-#include <map>
-#include <sstream>
-#include <thread>
-#include <future>
-#include <chrono>
#include <regex>
#include <queue>
-#include <climits>
-#include <sys/time.h>
-#include <sys/wait.h>
#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/utsname.h>
-#include <sys/resource.h>
#include <sys/socket.h>
#include <sys/un.h>
-#include <fcntl.h>
#include <netdb.h>
-#include <unistd.h>
-#include <errno.h>
-#include <cstring>
+#include <fcntl.h>
#include <termios.h>
-#include <poll.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <sys/utsname.h>
+#include <sys/resource.h>
-#include <pwd.h>
-#include <grp.h>
+#if HAVE_STATVFS
+#include <sys/statvfs.h>
+#endif
/* Includes required for chroot support. */
#if __linux__
@@ -67,424 +50,13 @@
#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old))
#endif
-#if HAVE_STATVFS
-#include <sys/statvfs.h>
-#endif
+#include <pwd.h>
+#include <grp.h>
#include <nlohmann/json.hpp>
-
namespace nix {
-using std::map;
-
-
-static string pathNullDevice = "/dev/null";
-
-
-/* Forward definition. */
-class Worker;
-struct HookInstance;
-
-
-/* A pointer to a goal. */
-struct Goal;
-class DerivationGoal;
-typedef std::shared_ptr<Goal> GoalPtr;
-typedef std::weak_ptr<Goal> WeakGoalPtr;
-
-struct CompareGoalPtrs {
- bool operator() (const GoalPtr & a, const GoalPtr & b) const;
-};
-
-/* Set of goals. */
-typedef set<GoalPtr, CompareGoalPtrs> Goals;
-typedef list<WeakGoalPtr> WeakGoals;
-
-/* A map of paths to goals (and the other way around). */
-typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
-
-
-
-struct Goal : public std::enable_shared_from_this<Goal>
-{
- typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
-
- /* Backlink to the worker. */
- Worker & worker;
-
- /* Goals that this goal is waiting for. */
- Goals waitees;
-
- /* Goals waiting for this one to finish. Must use weak pointers
- here to prevent cycles. */
- WeakGoals waiters;
-
- /* Number of goals we are/were waiting for that have failed. */
- unsigned int nrFailed;
-
- /* Number of substitution goals we are/were waiting for that
- failed because there are no substituters. */
- unsigned int nrNoSubstituters;
-
- /* Number of substitution goals we are/were waiting for that
- failed because othey had unsubstitutable references. */
- unsigned int nrIncompleteClosure;
-
- /* Name of this goal for debugging purposes. */
- string name;
-
- /* Whether the goal is finished. */
- ExitCode exitCode;
-
- /* Exception containing an error message, if any. */
- std::optional<Error> ex;
-
- Goal(Worker & worker) : worker(worker)
- {
- nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
- exitCode = ecBusy;
- }
-
- virtual ~Goal()
- {
- trace("goal destroyed");
- }
-
- virtual void work() = 0;
-
- void addWaitee(GoalPtr waitee);
-
- virtual void waiteeDone(GoalPtr waitee, ExitCode result);
-
- virtual void handleChildOutput(int fd, const string & data)
- {
- abort();
- }
-
- virtual void handleEOF(int fd)
- {
- abort();
- }
-
- void trace(const FormatOrString & fs);
-
- string getName()
- {
- return name;
- }
-
- /* Callback in case of a timeout. It should wake up its waiters,
- get rid of any running child processes that are being monitored
- by the worker (important!), etc. */
- virtual void timedOut(Error && ex) = 0;
-
- virtual string key() = 0;
-
- void amDone(ExitCode result, std::optional<Error> ex = {});
-};
-
-
-bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
- string s1 = a->key();
- string s2 = b->key();
- return s1 < s2;
-}
-
-
-typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
-
-
-/* A mapping used to remember for each child process to what goal it
- belongs, and file descriptors for receiving log data and output
- path creation commands. */
-struct Child
-{
- WeakGoalPtr goal;
- Goal * goal2; // ugly hackery
- set<int> fds;
- bool respectTimeouts;
- bool inBuildSlot;
- steady_time_point lastOutput; /* time we last got output on stdout/stderr */
- steady_time_point timeStarted;
-};
-
-
-/* The worker class. */
-class Worker
-{
-private:
-
- /* Note: the worker should only have strong pointers to the
- top-level goals. */
-
- /* The top-level goals of the worker. */
- Goals topGoals;
-
- /* Goals that are ready to do some work. */
- WeakGoals awake;
-
- /* Goals waiting for a build slot. */
- WeakGoals wantingToBuild;
-
- /* Child processes currently running. */
- std::list<Child> children;
-
- /* Number of build slots occupied. This includes local builds and
- substitutions but not remote builds via the build hook. */
- unsigned int nrLocalBuilds;
-
- /* Maps used to prevent multiple instantiations of a goal for the
- same derivation / path. */
- WeakGoalMap derivationGoals;
- WeakGoalMap substitutionGoals;
-
- /* Goals waiting for busy paths to be unlocked. */
- WeakGoals waitingForAnyGoal;
-
- /* Goals sleeping for a few seconds (polling a lock). */
- WeakGoals waitingForAWhile;
-
- /* Last time the goals in `waitingForAWhile' where woken up. */
- steady_time_point lastWokenUp;
-
- /* Cache for pathContentsGood(). */
- std::map<StorePath, bool> pathContentsGoodCache;
-
-public:
-
- const Activity act;
- const Activity actDerivations;
- const Activity actSubstitutions;
-
- /* Set if at least one derivation had a BuildError (i.e. permanent
- failure). */
- bool permanentFailure;
-
- /* Set if at least one derivation had a timeout. */
- bool timedOut;
-
- /* Set if at least one derivation fails with a hash mismatch. */
- bool hashMismatch;
-
- /* Set if at least one derivation is not deterministic in check mode. */
- bool checkMismatch;
-
- LocalStore & store;
-
- std::unique_ptr<HookInstance> hook;
-
- uint64_t expectedBuilds = 0;
- uint64_t doneBuilds = 0;
- uint64_t failedBuilds = 0;
- uint64_t runningBuilds = 0;
-
- uint64_t expectedSubstitutions = 0;
- uint64_t doneSubstitutions = 0;
- uint64_t failedSubstitutions = 0;
- uint64_t runningSubstitutions = 0;
- uint64_t expectedDownloadSize = 0;
- uint64_t doneDownloadSize = 0;
- uint64_t expectedNarSize = 0;
- uint64_t doneNarSize = 0;
-
- /* Whether to ask the build hook if it can build a derivation. If
- it answers with "decline-permanently", we don't try again. */
- bool tryBuildHook = true;
-
- Worker(LocalStore & store);
- ~Worker();
-
- /* Make a goal (with caching). */
-
- /* derivation goal */
-private:
- std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
- const StorePath & drvPath, const StringSet & wantedOutputs,
- std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
-public:
- std::shared_ptr<DerivationGoal> makeDerivationGoal(
- const StorePath & drvPath,
- const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
- std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
- const StorePath & drvPath, const BasicDerivation & drv,
- const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
-
- /* substitution goal */
- GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
-
- /* Remove a dead goal. */
- void removeGoal(GoalPtr goal);
-
- /* Wake up a goal (i.e., there is something for it to do). */
- void wakeUp(GoalPtr goal);
-
- /* Return the number of local build and substitution processes
- currently running (but not remote builds via the build
- hook). */
- unsigned int getNrLocalBuilds();
-
- /* Registers a running child process. `inBuildSlot' means that
- the process counts towards the jobs limit. */
- void childStarted(GoalPtr goal, const set<int> & fds,
- bool inBuildSlot, bool respectTimeouts);
-
- /* Unregisters a running child process. `wakeSleepers' should be
- false if there is no sense in waking up goals that are sleeping
- because they can't run yet (e.g., there is no free build slot,
- or the hook would still say `postpone'). */
- void childTerminated(Goal * goal, bool wakeSleepers = true);
-
- /* Put `goal' to sleep until a build slot becomes available (which
- might be right away). */
- void waitForBuildSlot(GoalPtr goal);
-
- /* Wait for any goal to finish. Pretty indiscriminate way to
- wait for some resource that some other goal is holding. */
- void waitForAnyGoal(GoalPtr goal);
-
- /* Wait for a few seconds and then retry this goal. Used when
- waiting for a lock held by another process. This kind of
- polling is inefficient, but POSIX doesn't really provide a way
- to wait for multiple locks in the main select() loop. */
- void waitForAWhile(GoalPtr goal);
-
- /* Loop until the specified top-level goals have finished. */
- void run(const Goals & topGoals);
-
- /* Wait for input to become available. */
- void waitForInput();
-
- unsigned int exitStatus();
-
- /* Check whether the given valid path exists and has the right
- contents. */
- bool pathContentsGood(const StorePath & path);
-
- void markContentsGood(const StorePath & path);
-
- void updateProgress()
- {
- actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds);
- actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions);
- act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize);
- act.setExpected(actCopyPath, expectedNarSize + doneNarSize);
- }
-};
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-void addToWeakGoals(WeakGoals & goals, GoalPtr p)
-{
- // FIXME: necessary?
- // FIXME: O(n)
- for (auto & i : goals)
- if (i.lock() == p) return;
- goals.push_back(p);
-}
-
-
-void Goal::addWaitee(GoalPtr waitee)
-{
- waitees.insert(waitee);
- addToWeakGoals(waitee->waiters, shared_from_this());
-}
-
-
-void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
-{
- assert(waitees.find(waitee) != waitees.end());
- waitees.erase(waitee);
-
- trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size()));
-
- if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed;
-
- if (result == ecNoSubstituters) ++nrNoSubstituters;
-
- if (result == ecIncompleteClosure) ++nrIncompleteClosure;
-
- if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) {
-
- /* If we failed and keepGoing is not set, we remove all
- remaining waitees. */
- for (auto & goal : waitees) {
- WeakGoals waiters2;
- for (auto & j : goal->waiters)
- if (j.lock() != shared_from_this()) waiters2.push_back(j);
- goal->waiters = waiters2;
- }
- waitees.clear();
-
- worker.wakeUp(shared_from_this());
- }
-}
-
-
-void Goal::amDone(ExitCode result, std::optional<Error> ex)
-{
- trace("done");
- assert(exitCode == ecBusy);
- assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure);
- exitCode = result;
-
- if (ex) {
- if (!waiters.empty())
- logError(ex->info());
- else
- this->ex = std::move(*ex);
- }
-
- for (auto & i : waiters) {
- GoalPtr goal = i.lock();
- if (goal) goal->waiteeDone(shared_from_this(), result);
- }
- waiters.clear();
- worker.removeGoal(shared_from_this());
-}
-
-
-void Goal::trace(const FormatOrString & fs)
-{
- debug("%1%: %2%", name, fs.s);
-}
-
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-/* Common initialisation performed in child processes. */
-static void commonChildInit(Pipe & logPipe)
-{
- restoreSignals();
-
- /* Put the child in a separate session (and thus a separate
- process group) so that it has no controlling terminal (meaning
- that e.g. ssh cannot open /dev/tty) and it doesn't receive
- terminal signals. */
- if (setsid() == -1)
- throw SysError("creating a new session");
-
- /* Dup the write side of the logger pipe into stderr. */
- if (dup2(logPipe.writeSide.get(), STDERR_FILENO) == -1)
- throw SysError("cannot pipe standard error into log file");
-
- /* Dup stderr to stdout. */
- if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1)
- throw SysError("cannot dup stderr into stdout");
-
- /* Reroute stdin to /dev/null. */
- int fdDevNull = open(pathNullDevice.c_str(), O_RDWR);
- if (fdDevNull == -1)
- throw SysError("cannot open '%1%'", pathNullDevice);
- if (dup2(fdDevNull, STDIN_FILENO) == -1)
- throw SysError("cannot dup null device into stdin");
- close(fdDevNull);
-}
-
void handleDiffHook(
uid_t uid, uid_t gid,
const Path & tryA, const Path & tryB,
@@ -517,594 +89,8 @@ void handleDiffHook(
}
}
-//////////////////////////////////////////////////////////////////////
-
-
-class UserLock
-{
-private:
- Path fnUserLock;
- AutoCloseFD fdUserLock;
-
- bool isEnabled = false;
- string user;
- uid_t uid = 0;
- gid_t gid = 0;
- std::vector<gid_t> supplementaryGIDs;
-
-public:
- UserLock();
-
- void kill();
-
- string getUser() { return user; }
- uid_t getUID() { assert(uid); return uid; }
- uid_t getGID() { assert(gid); return gid; }
- std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; }
-
- bool findFreeUser();
-
- bool enabled() { return isEnabled; }
-
-};
-
-
-UserLock::UserLock()
-{
- assert(settings.buildUsersGroup != "");
- createDirs(settings.nixStateDir + "/userpool");
-}
-
-bool UserLock::findFreeUser() {
- if (enabled()) return true;
-
- /* Get the members of the build-users-group. */
- struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
- if (!gr)
- throw Error("the group '%1%' specified in 'build-users-group' does not exist",
- settings.buildUsersGroup);
- gid = gr->gr_gid;
-
- /* Copy the result of getgrnam. */
- Strings users;
- for (char * * p = gr->gr_mem; *p; ++p) {
- debug("found build user '%1%'", *p);
- users.push_back(*p);
- }
-
- if (users.empty())
- throw Error("the build users group '%1%' has no members",
- settings.buildUsersGroup);
-
- /* Find a user account that isn't currently in use for another
- build. */
- for (auto & i : users) {
- debug("trying user '%1%'", i);
-
- struct passwd * pw = getpwnam(i.c_str());
- if (!pw)
- throw Error("the user '%1%' in the group '%2%' does not exist",
- i, settings.buildUsersGroup);
-
-
- fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str();
-
- AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
- if (!fd)
- throw SysError("opening user lock '%1%'", fnUserLock);
-
- if (lockFile(fd.get(), ltWrite, false)) {
- fdUserLock = std::move(fd);
- user = i;
- uid = pw->pw_uid;
-
- /* Sanity check... */
- if (uid == getuid() || uid == geteuid())
- throw Error("the Nix user should not be a member of '%1%'",
- settings.buildUsersGroup);
-
-#if __linux__
- /* Get the list of supplementary groups of this build user. This
- is usually either empty or contains a group such as "kvm". */
- supplementaryGIDs.resize(10);
- int ngroups = supplementaryGIDs.size();
- int err = getgrouplist(pw->pw_name, pw->pw_gid,
- supplementaryGIDs.data(), &ngroups);
- if (err == -1)
- throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name);
-
- supplementaryGIDs.resize(ngroups);
-#endif
-
- isEnabled = true;
- return true;
- }
- }
-
- return false;
-}
-
-void UserLock::kill()
-{
- killUser(uid);
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-struct HookInstance
-{
- /* Pipes for talking to the build hook. */
- Pipe toHook;
-
- /* Pipe for the hook's standard output/error. */
- Pipe fromHook;
-
- /* Pipe for the builder's standard output/error. */
- Pipe builderOut;
-
- /* The process ID of the hook. */
- Pid pid;
-
- FdSink sink;
-
- std::map<ActivityId, Activity> activities;
-
- HookInstance();
-
- ~HookInstance();
-};
-
-
-HookInstance::HookInstance()
-{
- debug("starting build hook '%s'", settings.buildHook);
-
- /* Create a pipe to get the output of the child. */
- fromHook.create();
-
- /* Create the communication pipes. */
- toHook.create();
-
- /* Create a pipe to get the output of the builder. */
- builderOut.create();
-
- /* Fork the hook. */
- pid = startProcess([&]() {
-
- commonChildInit(fromHook);
-
- if (chdir("/") == -1) throw SysError("changing into /");
-
- /* Dup the communication pipes. */
- if (dup2(toHook.readSide.get(), STDIN_FILENO) == -1)
- throw SysError("dupping to-hook read side");
-
- /* Use fd 4 for the builder's stdout/stderr. */
- if (dup2(builderOut.writeSide.get(), 4) == -1)
- throw SysError("dupping builder's stdout/stderr");
-
- /* Hack: pass the read side of that fd to allow build-remote
- to read SSH error messages. */
- if (dup2(builderOut.readSide.get(), 5) == -1)
- throw SysError("dupping builder's stdout/stderr");
-
- Strings args = {
- std::string(baseNameOf(settings.buildHook.get())),
- std::to_string(verbosity),
- };
-
- execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data());
-
- throw SysError("executing '%s'", settings.buildHook);
- });
-
- pid.setSeparatePG(true);
- fromHook.writeSide = -1;
- toHook.readSide = -1;
-
- sink = FdSink(toHook.writeSide.get());
- std::map<std::string, Config::SettingInfo> settings;
- globalConfig.getSettings(settings);
- for (auto & setting : settings)
- sink << 1 << setting.first << setting.second.value;
- sink << 0;
-}
-
-
-HookInstance::~HookInstance()
-{
- try {
- toHook.writeSide = -1;
- if (pid != -1) pid.kill();
- } catch (...) {
- ignoreException();
- }
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
-
-class SubstitutionGoal;
-
-/* Unless we are repairing, we don't both to test validity and just assume it,
- so the choices are `Absent` or `Valid`. */
-enum struct PathStatus {
- Corrupt,
- Absent,
- Valid,
-};
-
-struct InitialOutputStatus {
- StorePath path;
- PathStatus status;
- /* Valid in the store, and additionally non-corrupt if we are repairing */
- bool isValid() const {
- return status == PathStatus::Valid;
- }
- /* Merely present, allowed to be corrupt */
- bool isPresent() const {
- return status == PathStatus::Corrupt
- || status == PathStatus::Valid;
- }
-};
-
-struct InitialOutput {
- bool wanted;
- std::optional<InitialOutputStatus> known;
-};
-
-class DerivationGoal : public Goal
-{
-private:
- /* Whether to use an on-disk .drv file. */
- bool useDerivation;
-
- /* The path of the derivation. */
- StorePath drvPath;
-
- /* The specific outputs that we need to build. Empty means all of
- them. */
- StringSet wantedOutputs;
-
- /* Whether additional wanted outputs have been added. */
- bool needRestart = false;
-
- /* Whether to retry substituting the outputs after building the
- inputs. */
- bool retrySubstitution;
-
- /* The derivation stored at drvPath. */
- std::unique_ptr<BasicDerivation> drv;
-
- std::unique_ptr<ParsedDerivation> parsedDrv;
-
- /* The remainder is state held during the build. */
-
- /* Locks on (fixed) output paths. */
- PathLocks outputLocks;
-
- /* All input paths (that is, the union of FS closures of the
- immediate input paths). */
- StorePathSet inputPaths;
-
- std::map<std::string, InitialOutput> initialOutputs;
-
- /* User selected for running the builder. */
- std::unique_ptr<UserLock> buildUser;
-
- /* The process ID of the builder. */
- Pid pid;
-
- /* The temporary directory. */
- Path tmpDir;
-
- /* The path of the temporary directory in the sandbox. */
- Path tmpDirInSandbox;
-
- /* File descriptor for the log file. */
- AutoCloseFD fdLogFile;
- std::shared_ptr<BufferedSink> logFileSink, logSink;
-
- /* Number of bytes received from the builder's stdout/stderr. */
- unsigned long logSize;
-
- /* The most recent log lines. */
- std::list<std::string> logTail;
-
- std::string currentLogLine;
- size_t currentLogLinePos = 0; // to handle carriage return
-
- std::string currentHookLine;
-
- /* Pipe for the builder's standard output/error. */
- Pipe builderOut;
-
- /* Pipe for synchronising updates to the builder namespaces. */
- Pipe userNamespaceSync;
-
- /* The mount namespace of the builder, used to add additional
- paths to the sandbox as a result of recursive Nix calls. */
- AutoCloseFD sandboxMountNamespace;
-
- /* On Linux, whether we're doing the build in its own user
- namespace. */
- bool usingUserNamespace = true;
-
- /* The build hook. */
- std::unique_ptr<HookInstance> hook;
-
- /* Whether we're currently doing a chroot build. */
- bool useChroot = false;
-
- Path chrootRootDir;
-
- /* RAII object to delete the chroot directory. */
- std::shared_ptr<AutoDelete> autoDelChroot;
-
- /* The sort of derivation we are building. */
- DerivationType derivationType;
-
- /* Whether to run the build in a private network namespace. */
- bool privateNetwork = false;
-
- typedef void (DerivationGoal::*GoalState)();
- GoalState state;
-
- /* Stuff we need to pass to initChild(). */
- struct ChrootPath {
- Path source;
- bool optional;
- ChrootPath(Path source = "", bool optional = false)
- : source(source), optional(optional)
- { }
- };
- typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
- DirsInChroot dirsInChroot;
-
- typedef map<string, string> Environment;
- Environment env;
-
-#if __APPLE__
- typedef string SandboxProfile;
- SandboxProfile additionalSandboxProfile;
-#endif
-
- /* Hash rewriting. */
- StringMap inputRewrites, outputRewrites;
- typedef map<StorePath, StorePath> RedirectedOutputs;
- RedirectedOutputs redirectedOutputs;
-
- /* The outputs paths used during the build.
-
- - Input-addressed derivations or fixed content-addressed outputs are
- sometimes built when some of their outputs already exist, and can not
- be hidden via sandboxing. We use temporary locations instead and
- rewrite after the build. Otherwise the regular predetermined paths are
- put here.
-
- - Floating content-addressed derivations do not know their final build
- output paths until the outputs are hashed, so random locations are
- used, and then renamed. The randomness helps guard against hidden
- self-references.
- */
- OutputPathMap scratchOutputs;
-
- /* The final output paths of the build.
-
- - For input-addressed derivations, always the precomputed paths
-
- - For content-addressed derivations, calcuated from whatever the hash
- ends up being. (Note that fixed outputs derivations that produce the
- "wrong" output still install that data under its true content-address.)
- */
- OutputPathMap finalOutputs;
-
- BuildMode buildMode;
-
- /* If we're repairing without a chroot, there may be outputs that
- are valid but corrupt. So we redirect these outputs to
- temporary paths. */
- StorePathSet redirectedBadOutputs;
-
- BuildResult result;
-
- /* The current round, if we're building multiple times. */
- size_t curRound = 1;
-
- size_t nrRounds;
-
- /* Path registration info from the previous round, if we're
- building multiple times. Since this contains the hash, it
- allows us to compare whether two rounds produced the same
- result. */
- std::map<Path, ValidPathInfo> prevInfos;
-
- uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
- gid_t sandboxGid() { return usingUserNamespace ? 100 : buildUser->getGID(); }
-
- const static Path homeDir;
-
- std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
-
- std::unique_ptr<Activity> act;
-
- /* Activity that denotes waiting for a lock. */
- std::unique_ptr<Activity> actLock;
-
- std::map<ActivityId, Activity> builderActivities;
-
- /* The remote machine on which we're building. */
- std::string machineName;
-
- /* The recursive Nix daemon socket. */
- AutoCloseFD daemonSocket;
-
- /* The daemon main thread. */
- std::thread daemonThread;
-
- /* The daemon worker threads. */
- std::vector<std::thread> daemonWorkerThreads;
-
- /* Paths that were added via recursive Nix calls. */
- StorePathSet addedPaths;
-
- /* Recursive Nix calls are only allowed to build or realize paths
- in the original input closure or added via a recursive Nix call
- (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
- /nix/store/<bla> is some arbitrary path in a binary cache). */
- bool isAllowed(const StorePath & path)
- {
- return inputPaths.count(path) || addedPaths.count(path);
- }
-
- friend struct RestrictedStore;
-
-public:
- DerivationGoal(const StorePath & drvPath,
- const StringSet & wantedOutputs, Worker & worker,
- BuildMode buildMode = bmNormal);
- DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
- const StringSet & wantedOutputs, Worker & worker,
- BuildMode buildMode = bmNormal);
- ~DerivationGoal();
-
- /* Whether we need to perform hash rewriting if there are valid output paths. */
- bool needsHashRewrite();
-
- void timedOut(Error && ex) override;
-
- string key() override
- {
- /* Ensure that derivations get built in order of their name,
- i.e. a derivation named "aardvark" always comes before
- "baboon". And substitution goals always happen before
- derivation goals (due to "b$"). */
- return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
- }
-
- void work() override;
-
- StorePath getDrvPath()
- {
- return drvPath;
- }
-
- /* Add wanted outputs to an already existing derivation goal. */
- void addWantedOutputs(const StringSet & outputs);
-
- BuildResult getResult() { return result; }
-
-private:
- /* The states. */
- void getDerivation();
- void loadDerivation();
- void haveDerivation();
- void outputsSubstitutionTried();
- void gaveUpOnSubstitution();
- void closureRepaired();
- void inputsRealised();
- void tryToBuild();
- void tryLocalBuild();
- void buildDone();
-
- void resolvedFinished();
-
- /* Is the build hook willing to perform the build? */
- HookReply tryBuildHook();
-
- /* Start building a derivation. */
- void startBuilder();
-
- /* Fill in the environment for the builder. */
- void initEnv();
-
- /* Setup tmp dir location. */
- void initTmpDir();
-
- /* Write a JSON file containing the derivation attributes. */
- void writeStructuredAttrs();
-
- void startDaemon();
-
- void stopDaemon();
-
- /* Add 'path' to the set of paths that may be referenced by the
- outputs, and make it appear in the sandbox. */
- void addDependency(const StorePath & path);
-
- /* Make a file owned by the builder. */
- void chownToBuilder(const Path & path);
-
- /* Run the builder's process. */
- void runChild();
-
- friend int childEntry(void *);
-
- /* Check that the derivation outputs all exist and register them
- as valid. */
- void registerOutputs();
-
- /* Check that an output meets the requirements specified by the
- 'outputChecks' attribute (or the legacy
- '{allowed,disallowed}{References,Requisites}' attributes). */
- void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
-
- /* Open a log file and a pipe to it. */
- Path openLogFile();
-
- /* Close the log file. */
- void closeLogFile();
-
- /* Delete the temporary directory, if we have one. */
- void deleteTmpDir(bool force);
-
- /* Callback used by the worker to write to the log. */
- void handleChildOutput(int fd, const string & data) override;
- void handleEOF(int fd) override;
- void flushLine();
-
- /* Wrappers around the corresponding Store methods that first consult the
- derivation. This is currently needed because when there is no drv file
- there also is no DB entry. */
- std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
- OutputPathMap queryDerivationOutputMap();
-
- /* Return the set of (in)valid paths. */
- void checkPathValidity();
-
- /* Forcibly kill the child process, if any. */
- void killChild();
-
- /* Create alternative path calculated from but distinct from the
- input, so we can avoid overwriting outputs (or other store paths)
- that already exist. */
- StorePath makeFallbackPath(const StorePath & path);
- /* Make a path to another based on the output name along with the
- derivation hash. */
- /* FIXME add option to randomize, so we can audit whether our
- rewrites caught everything */
- StorePath makeFallbackPath(std::string_view outputName);
-
- void repairClosure();
-
- void started();
-
- void done(
- BuildResult::Status status,
- std::optional<Error> ex = {});
-
- StorePathSet exportReferences(const StorePathSet & storePaths);
-};
-
-
const Path DerivationGoal::homeDir = "/homeless-shelter";
-
DerivationGoal::DerivationGoal(const StorePath & drvPath,
const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
: Goal(worker)
@@ -1159,6 +145,16 @@ DerivationGoal::~DerivationGoal()
}
+string DerivationGoal::key()
+{
+ /* Ensure that derivations get built in order of their name,
+ i.e. a derivation named "aardvark" always comes before
+ "baboon". And substitution goals always happen before
+ derivation goals (due to "b$"). */
+ return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
+}
+
+
inline bool DerivationGoal::needsHashRewrite()
{
#if __linux__
@@ -2424,12 +1420,6 @@ void DerivationGoal::startBuilder()
Samba-in-QEMU. */
createDirs(chrootRootDir + "/etc");
- writeFile(chrootRootDir + "/etc/passwd", fmt(
- "root:x:0:0:Nix build user:%3%:/noshell\n"
- "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n"
- "nobody:x:65534:65534:Nobody:/:/noshell\n",
- sandboxUid(), sandboxGid(), settings.sandboxBuildDir));
-
/* Declare the build user's group so that programs get a consistent
view of the system (e.g., "id -gn"). */
writeFile(chrootRootDir + "/etc/group",
@@ -2734,6 +1724,14 @@ void DerivationGoal::startBuilder()
throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces");
}
+ /* Now that we now the sandbox uid, we can write
+ /etc/passwd. */
+ writeFile(chrootRootDir + "/etc/passwd", fmt(
+ "root:x:0:0:Nix build user:%3%:/noshell\n"
+ "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n"
+ "nobody:x:65534:65534:Nobody:/:/noshell\n",
+ sandboxUid(), sandboxGid(), settings.sandboxBuildDir));
+
/* Save the mount namespace of the child. We have to do this
*before* the child does a chroot. */
sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY);
@@ -4740,944 +3738,4 @@ void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
}
-//////////////////////////////////////////////////////////////////////
-
-
-class SubstitutionGoal : public Goal
-{
- friend class Worker;
-
-private:
- /* The store path that should be realised through a substitute. */
- StorePath storePath;
-
- /* The path the substituter refers to the path as. This will be
- * different when the stores have different names. */
- std::optional<StorePath> subPath;
-
- /* The remaining substituters. */
- std::list<ref<Store>> subs;
-
- /* The current substituter. */
- std::shared_ptr<Store> sub;
-
- /* Whether a substituter failed. */
- bool substituterFailed = false;
-
- /* Path info returned by the substituter's query info operation. */
- std::shared_ptr<const ValidPathInfo> info;
-
- /* Pipe for the substituter's standard output. */
- Pipe outPipe;
-
- /* The substituter thread. */
- std::thread thr;
-
- std::promise<void> promise;
-
- /* Whether to try to repair a valid path. */
- RepairFlag repair;
-
- /* Location where we're downloading the substitute. Differs from
- storePath when doing a repair. */
- Path destPath;
-
- std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
- maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
-
- typedef void (SubstitutionGoal::*GoalState)();
- GoalState state;
-
- /* Content address for recomputing store path */
- std::optional<ContentAddress> ca;
-
-public:
- SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
- ~SubstitutionGoal();
-
- void timedOut(Error && ex) override { abort(); };
-
- string key() override
- {
- /* "a$" ensures substitution goals happen before derivation
- goals. */
- return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
- }
-
- void work() override;
-
- /* The states. */
- void init();
- void tryNext();
- void gotInfo();
- void referencesValid();
- void tryToRun();
- void finished();
-
- /* Callback used by the worker to write to the log. */
- void handleChildOutput(int fd, const string & data) override;
- void handleEOF(int fd) override;
-
- StorePath getStorePath() { return storePath; }
-};
-
-
-SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
- : Goal(worker)
- , storePath(storePath)
- , repair(repair)
- , ca(ca)
-{
- state = &SubstitutionGoal::init;
- name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
- trace("created");
- maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions);
-}
-
-
-SubstitutionGoal::~SubstitutionGoal()
-{
- try {
- if (thr.joinable()) {
- // FIXME: signal worker thread to quit.
- thr.join();
- worker.childTerminated(this);
- }
- } catch (...) {
- ignoreException();
- }
-}
-
-
-void SubstitutionGoal::work()
-{
- (this->*state)();
-}
-
-
-void SubstitutionGoal::init()
-{
- trace("init");
-
- worker.store.addTempRoot(storePath);
-
- /* If the path already exists we're done. */
- if (!repair && worker.store.isValidPath(storePath)) {
- amDone(ecSuccess);
- return;
- }
-
- if (settings.readOnlyMode)
- throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath));
-
- subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
-
- tryNext();
-}
-
-
-void SubstitutionGoal::tryNext()
-{
- trace("trying next substituter");
-
- if (subs.size() == 0) {
- /* None left. Terminate this goal and let someone else deal
- with it. */
- debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath));
-
- /* Hack: don't indicate failure if there were no substituters.
- In that case the calling derivation should just do a
- build. */
- amDone(substituterFailed ? ecFailed : ecNoSubstituters);
-
- if (substituterFailed) {
- worker.failedSubstitutions++;
- worker.updateProgress();
- }
-
- return;
- }
-
- sub = subs.front();
- subs.pop_front();
-
- if (ca) {
- subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
- if (sub->storeDir == worker.store.storeDir)
- assert(subPath == storePath);
- } else if (sub->storeDir != worker.store.storeDir) {
- tryNext();
- return;
- }
-
- try {
- // FIXME: make async
- info = sub->queryPathInfo(subPath ? *subPath : storePath);
- } catch (InvalidPath &) {
- tryNext();
- return;
- } catch (SubstituterDisabled &) {
- if (settings.tryFallback) {
- tryNext();
- return;
- }
- throw;
- } catch (Error & e) {
- if (settings.tryFallback) {
- logError(e.info());
- tryNext();
- return;
- }
- throw;
- }
-
- if (info->path != storePath) {
- if (info->isContentAddressed(*sub) && info->references.empty()) {
- auto info2 = std::make_shared<ValidPathInfo>(*info);
- info2->path = storePath;
- info = info2;
- } else {
- printError("asked '%s' for '%s' but got '%s'",
- sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path));
- tryNext();
- return;
- }
- }
-
- /* Update the total expected download size. */
- auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
-
- maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, info->narSize);
-
- maintainExpectedDownload =
- narInfo && narInfo->fileSize
- ? std::make_unique<MaintainCount<uint64_t>>(worker.expectedDownloadSize, narInfo->fileSize)
- : nullptr;
-
- worker.updateProgress();
-
- /* Bail out early if this substituter lacks a valid
- signature. LocalStore::addToStore() also checks for this, but
- only after we've downloaded the path. */
- if (worker.store.requireSigs
- && !sub->isTrusted
- && !info->checkSignatures(worker.store, worker.store.getPublicKeys()))
- {
- logWarning({
- .name = "Invalid path signature",
- .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'",
- sub->getUri(), worker.store.printStorePath(storePath))
- });
- tryNext();
- return;
- }
-
- /* To maintain the closure invariant, we first have to realise the
- paths referenced by this one. */
- for (auto & i : info->references)
- if (i != storePath) /* ignore self-references */
- addWaitee(worker.makeSubstitutionGoal(i));
-
- if (waitees.empty()) /* to prevent hang (no wake-up event) */
- referencesValid();
- else
- state = &SubstitutionGoal::referencesValid;
-}
-
-
-void SubstitutionGoal::referencesValid()
-{
- trace("all references realised");
-
- if (nrFailed > 0) {
- debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath));
- amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
- return;
- }
-
- for (auto & i : info->references)
- if (i != storePath) /* ignore self-references */
- assert(worker.store.isValidPath(i));
-
- state = &SubstitutionGoal::tryToRun;
- worker.wakeUp(shared_from_this());
-}
-
-
-void SubstitutionGoal::tryToRun()
-{
- trace("trying to run");
-
- /* Make sure that we are allowed to start a build. Note that even
- if maxBuildJobs == 0 (no local builds allowed), we still allow
- a substituter to run. This is because substitutions cannot be
- distributed to another machine via the build hook. */
- if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) {
- worker.waitForBuildSlot(shared_from_this());
- return;
- }
-
- maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
- worker.updateProgress();
-
- outPipe.create();
-
- promise = std::promise<void>();
-
- thr = std::thread([this]() {
- try {
- /* Wake up the worker loop when we're done. */
- Finally updateStats([this]() { outPipe.writeSide = -1; });
-
- Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()});
- PushActivity pact(act.id);
-
- copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()),
- subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
-
- promise.set_value();
- } catch (...) {
- promise.set_exception(std::current_exception());
- }
- });
-
- worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false);
-
- state = &SubstitutionGoal::finished;
-}
-
-
-void SubstitutionGoal::finished()
-{
- trace("substitute finished");
-
- thr.join();
- worker.childTerminated(this);
-
- try {
- promise.get_future().get();
- } catch (std::exception & e) {
- printError(e.what());
-
- /* Cause the parent build to fail unless --fallback is given,
- or the substitute has disappeared. The latter case behaves
- the same as the substitute never having existed in the
- first place. */
- try {
- throw;
- } catch (SubstituteGone &) {
- } catch (...) {
- substituterFailed = true;
- }
-
- /* Try the next substitute. */
- state = &SubstitutionGoal::tryNext;
- worker.wakeUp(shared_from_this());
- return;
- }
-
- worker.markContentsGood(storePath);
-
- printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath));
-
- maintainRunningSubstitutions.reset();
-
- maintainExpectedSubstitutions.reset();
- worker.doneSubstitutions++;
-
- if (maintainExpectedDownload) {
- auto fileSize = maintainExpectedDownload->delta;
- maintainExpectedDownload.reset();
- worker.doneDownloadSize += fileSize;
- }
-
- worker.doneNarSize += maintainExpectedNar->delta;
- maintainExpectedNar.reset();
-
- worker.updateProgress();
-
- amDone(ecSuccess);
-}
-
-
-void SubstitutionGoal::handleChildOutput(int fd, const string & data)
-{
-}
-
-
-void SubstitutionGoal::handleEOF(int fd)
-{
- if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
-}
-
-//////////////////////////////////////////////////////////////////////
-
-
-Worker::Worker(LocalStore & store)
- : act(*logger, actRealise)
- , actDerivations(*logger, actBuilds)
- , actSubstitutions(*logger, actCopyPaths)
- , store(store)
-{
- /* Debugging: prevent recursive workers. */
- nrLocalBuilds = 0;
- lastWokenUp = steady_time_point::min();
- permanentFailure = false;
- timedOut = false;
- hashMismatch = false;
- checkMismatch = false;
-}
-
-
-Worker::~Worker()
-{
- /* Explicitly get rid of all strong pointers now. After this all
- goals that refer to this worker should be gone. (Otherwise we
- are in trouble, since goals may call childTerminated() etc. in
- their destructors). */
- topGoals.clear();
-
- assert(expectedSubstitutions == 0);
- assert(expectedDownloadSize == 0);
- assert(expectedNarSize == 0);
-}
-
-
-std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
- const StorePath & drvPath,
- const StringSet & wantedOutputs,
- std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal)
-{
- WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath];
- GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME
- std::shared_ptr<DerivationGoal> goal;
- if (!abstract_goal) {
- goal = mkDrvGoal();
- abstract_goal_weak = goal;
- wakeUp(goal);
- } else {
- goal = std::dynamic_pointer_cast<DerivationGoal>(abstract_goal);
- assert(goal);
- goal->addWantedOutputs(wantedOutputs);
- }
- return goal;
-}
-
-
-std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
- const StringSet & wantedOutputs, BuildMode buildMode)
-{
- return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
- return std::make_shared<DerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
- });
-}
-
-
-std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
- const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode)
-{
- return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
- return std::make_shared<DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
- });
-}
-
-
-GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
-{
- WeakGoalPtr & goal_weak = substitutionGoals[path];
- GoalPtr goal = goal_weak.lock(); // FIXME
- if (!goal) {
- goal = std::make_shared<SubstitutionGoal>(path, *this, repair, ca);
- goal_weak = goal;
- wakeUp(goal);
- }
- return goal;
-}
-
-
-static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap)
-{
- /* !!! inefficient */
- for (WeakGoalMap::iterator i = goalMap.begin();
- i != goalMap.end(); )
- if (i->second.lock() == goal) {
- WeakGoalMap::iterator j = i; ++j;
- goalMap.erase(i);
- i = j;
- }
- else ++i;
-}
-
-
-void Worker::removeGoal(GoalPtr goal)
-{
- nix::removeGoal(goal, derivationGoals);
- nix::removeGoal(goal, substitutionGoals);
- if (topGoals.find(goal) != topGoals.end()) {
- topGoals.erase(goal);
- /* If a top-level goal failed, then kill all other goals
- (unless keepGoing was set). */
- if (goal->exitCode == Goal::ecFailed && !settings.keepGoing)
- topGoals.clear();
- }
-
- /* Wake up goals waiting for any goal to finish. */
- for (auto & i : waitingForAnyGoal) {
- GoalPtr goal = i.lock();
- if (goal) wakeUp(goal);
- }
-
- waitingForAnyGoal.clear();
-}
-
-
-void Worker::wakeUp(GoalPtr goal)
-{
- goal->trace("woken up");
- addToWeakGoals(awake, goal);
-}
-
-
-unsigned Worker::getNrLocalBuilds()
-{
- return nrLocalBuilds;
-}
-
-
-void Worker::childStarted(GoalPtr goal, const set<int> & fds,
- bool inBuildSlot, bool respectTimeouts)
-{
- Child child;
- child.goal = goal;
- child.goal2 = goal.get();
- child.fds = fds;
- child.timeStarted = child.lastOutput = steady_time_point::clock::now();
- child.inBuildSlot = inBuildSlot;
- child.respectTimeouts = respectTimeouts;
- children.emplace_back(child);
- if (inBuildSlot) nrLocalBuilds++;
-}
-
-
-void Worker::childTerminated(Goal * goal, bool wakeSleepers)
-{
- auto i = std::find_if(children.begin(), children.end(),
- [&](const Child & child) { return child.goal2 == goal; });
- if (i == children.end()) return;
-
- if (i->inBuildSlot) {
- assert(nrLocalBuilds > 0);
- nrLocalBuilds--;
- }
-
- children.erase(i);
-
- if (wakeSleepers) {
-
- /* Wake up goals waiting for a build slot. */
- for (auto & j : wantingToBuild) {
- GoalPtr goal = j.lock();
- if (goal) wakeUp(goal);
- }
-
- wantingToBuild.clear();
- }
-}
-
-
-void Worker::waitForBuildSlot(GoalPtr goal)
-{
- debug("wait for build slot");
- if (getNrLocalBuilds() < settings.maxBuildJobs)
- wakeUp(goal); /* we can do it right away */
- else
- addToWeakGoals(wantingToBuild, goal);
-}
-
-
-void Worker::waitForAnyGoal(GoalPtr goal)
-{
- debug("wait for any goal");
- addToWeakGoals(waitingForAnyGoal, goal);
-}
-
-
-void Worker::waitForAWhile(GoalPtr goal)
-{
- debug("wait for a while");
- addToWeakGoals(waitingForAWhile, goal);
-}
-
-
-void Worker::run(const Goals & _topGoals)
-{
- for (auto & i : _topGoals) topGoals.insert(i);
-
- debug("entered goal loop");
-
- while (1) {
-
- checkInterrupt();
-
- store.autoGC(false);
-
- /* Call every wake goal (in the ordering established by
- CompareGoalPtrs). */
- while (!awake.empty() && !topGoals.empty()) {
- Goals awake2;
- for (auto & i : awake) {
- GoalPtr goal = i.lock();
- if (goal) awake2.insert(goal);
- }
- awake.clear();
- for (auto & goal : awake2) {
- checkInterrupt();
- goal->work();
- if (topGoals.empty()) break; // stuff may have been cancelled
- }
- }
-
- if (topGoals.empty()) break;
-
- /* Wait for input. */
- if (!children.empty() || !waitingForAWhile.empty())
- waitForInput();
- else {
- if (awake.empty() && 0 == settings.maxBuildJobs)
- {
- if (getMachines().empty())
- throw Error("unable to start any build; either increase '--max-jobs' "
- "or enable remote builds."
- "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
- else
- throw Error("unable to start any build; remote machines may not have "
- "all required system features."
- "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
-
- }
- assert(!awake.empty());
- }
- }
-
- /* If --keep-going is not set, it's possible that the main goal
- exited while some of its subgoals were still active. But if
- --keep-going *is* set, then they must all be finished now. */
- assert(!settings.keepGoing || awake.empty());
- assert(!settings.keepGoing || wantingToBuild.empty());
- assert(!settings.keepGoing || children.empty());
-}
-
-void Worker::waitForInput()
-{
- printMsg(lvlVomit, "waiting for children");
-
- /* Process output from the file descriptors attached to the
- children, namely log output and output path creation commands.
- We also use this to detect child termination: if we get EOF on
- the logger pipe of a build, we assume that the builder has
- terminated. */
-
- bool useTimeout = false;
- long timeout = 0;
- auto before = steady_time_point::clock::now();
-
- /* If we're monitoring for silence on stdout/stderr, or if there
- is a build timeout, then wait for input until the first
- deadline for any child. */
- auto nearest = steady_time_point::max(); // nearest deadline
- if (settings.minFree.get() != 0)
- // Periodicallty wake up to see if we need to run the garbage collector.
- nearest = before + std::chrono::seconds(10);
- for (auto & i : children) {
- if (!i.respectTimeouts) continue;
- if (0 != settings.maxSilentTime)
- nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime));
- if (0 != settings.buildTimeout)
- nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout));
- }
- if (nearest != steady_time_point::max()) {
- timeout = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count());
- useTimeout = true;
- }
-
- /* If we are polling goals that are waiting for a lock, then wake
- up after a few seconds at most. */
- if (!waitingForAWhile.empty()) {
- useTimeout = true;
- if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before;
- timeout = std::max(1L,
- (long) std::chrono::duration_cast<std::chrono::seconds>(
- lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count());
- } else lastWokenUp = steady_time_point::min();
-
- if (useTimeout)
- vomit("sleeping %d seconds", timeout);
-
- /* Use select() to wait for the input side of any logger pipe to
- become `available'. Note that `available' (i.e., non-blocking)
- includes EOF. */
- std::vector<struct pollfd> pollStatus;
- std::map <int, int> fdToPollStatus;
- for (auto & i : children) {
- for (auto & j : i.fds) {
- pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
- fdToPollStatus[j] = pollStatus.size() - 1;
- }
- }
-
- if (poll(pollStatus.data(), pollStatus.size(),
- useTimeout ? timeout * 1000 : -1) == -1) {
- if (errno == EINTR) return;
- throw SysError("waiting for input");
- }
-
- auto after = steady_time_point::clock::now();
-
- /* Process all available file descriptors. FIXME: this is
- O(children * fds). */
- decltype(children)::iterator i;
- for (auto j = children.begin(); j != children.end(); j = i) {
- i = std::next(j);
-
- checkInterrupt();
-
- GoalPtr goal = j->goal.lock();
- assert(goal);
-
- set<int> fds2(j->fds);
- std::vector<unsigned char> buffer(4096);
- for (auto & k : fds2) {
- if (pollStatus.at(fdToPollStatus.at(k)).revents) {
- ssize_t rd = ::read(k, buffer.data(), buffer.size());
- // FIXME: is there a cleaner way to handle pt close
- // than EIO? Is this even standard?
- if (rd == 0 || (rd == -1 && errno == EIO)) {
- debug("%1%: got EOF", goal->getName());
- goal->handleEOF(k);
- j->fds.erase(k);
- } else if (rd == -1) {
- if (errno != EINTR)
- throw SysError("%s: read failed", goal->getName());
- } else {
- printMsg(lvlVomit, "%1%: read %2% bytes",
- goal->getName(), rd);
- string data((char *) buffer.data(), rd);
- j->lastOutput = after;
- goal->handleChildOutput(k, data);
- }
- }
- }
-
- if (goal->exitCode == Goal::ecBusy &&
- 0 != settings.maxSilentTime &&
- j->respectTimeouts &&
- after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime))
- {
- goal->timedOut(Error(
- "%1% timed out after %2% seconds of silence",
- goal->getName(), settings.maxSilentTime));
- }
-
- else if (goal->exitCode == Goal::ecBusy &&
- 0 != settings.buildTimeout &&
- j->respectTimeouts &&
- after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout))
- {
- goal->timedOut(Error(
- "%1% timed out after %2% seconds",
- goal->getName(), settings.buildTimeout));
- }
- }
-
- if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) {
- lastWokenUp = after;
- for (auto & i : waitingForAWhile) {
- GoalPtr goal = i.lock();
- if (goal) wakeUp(goal);
- }
- waitingForAWhile.clear();
- }
-}
-
-
-unsigned int Worker::exitStatus()
-{
- /*
- * 1100100
- * ^^^^
- * |||`- timeout
- * ||`-- output hash mismatch
- * |`--- build failure
- * `---- not deterministic
- */
- unsigned int mask = 0;
- bool buildFailure = permanentFailure || timedOut || hashMismatch;
- if (buildFailure)
- mask |= 0x04; // 100
- if (timedOut)
- mask |= 0x01; // 101
- if (hashMismatch)
- mask |= 0x02; // 102
- if (checkMismatch) {
- mask |= 0x08; // 104
- }
-
- if (mask)
- mask |= 0x60;
- return mask ? mask : 1;
-}
-
-
-bool Worker::pathContentsGood(const StorePath & path)
-{
- auto i = pathContentsGoodCache.find(path);
- if (i != pathContentsGoodCache.end()) return i->second;
- printInfo("checking path '%s'...", store.printStorePath(path));
- auto info = store.queryPathInfo(path);
- bool res;
- if (!pathExists(store.printStorePath(path)))
- res = false;
- else {
- HashResult current = hashPath(info->narHash.type, store.printStorePath(path));
- Hash nullHash(htSHA256);
- res = info->narHash == nullHash || info->narHash == current.first;
- }
- pathContentsGoodCache.insert_or_assign(path, res);
- if (!res)
- logError({
- .name = "Corrupted path",
- .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path))
- });
- return res;
-}
-
-
-void Worker::markContentsGood(const StorePath & path)
-{
- pathContentsGoodCache.insert_or_assign(path, true);
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-static void primeCache(Store & store, const std::vector<StorePathWithOutputs> & paths)
-{
- StorePathSet willBuild, willSubstitute, unknown;
- uint64_t downloadSize, narSize;
- store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
-
- if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty())
- throw Error(
- "%d derivations need to be built, but neither local builds ('--max-jobs') "
- "nor remote builds ('--builders') are enabled", willBuild.size());
-}
-
-
-void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
-{
- Worker worker(*this);
-
- primeCache(*this, drvPaths);
-
- Goals goals;
- for (auto & path : drvPaths) {
- if (path.path.isDerivation())
- goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode));
- else
- goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair));
- }
-
- worker.run(goals);
-
- StorePathSet failed;
- std::optional<Error> ex;
- for (auto & i : goals) {
- if (i->ex) {
- if (ex)
- logError(i->ex->info());
- else
- ex = i->ex;
- }
- if (i->exitCode != Goal::ecSuccess) {
- DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get());
- if (i2) failed.insert(i2->getDrvPath());
- else failed.insert(dynamic_cast<SubstitutionGoal *>(i.get())->getStorePath());
- }
- }
-
- if (failed.size() == 1 && ex) {
- ex->status = worker.exitStatus();
- throw *ex;
- } else if (!failed.empty()) {
- if (ex) logError(ex->info());
- throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed));
- }
-}
-
-BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
- BuildMode buildMode)
-{
- Worker worker(*this);
- auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode);
-
- BuildResult result;
-
- try {
- worker.run(Goals{goal});
- result = goal->getResult();
- } catch (Error & e) {
- result.status = BuildResult::MiscFailure;
- result.errorMsg = e.msg();
- }
-
- return result;
-}
-
-
-void LocalStore::ensurePath(const StorePath & path)
-{
- /* If the path is already valid, we're done. */
- if (isValidPath(path)) return;
-
- primeCache(*this, {{path}});
-
- Worker worker(*this);
- GoalPtr goal = worker.makeSubstitutionGoal(path);
- Goals goals = {goal};
-
- worker.run(goals);
-
- if (goal->exitCode != Goal::ecSuccess) {
- if (goal->ex) {
- goal->ex->status = worker.exitStatus();
- throw *goal->ex;
- } else
- throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path));
- }
-}
-
-
-void LocalStore::repairPath(const StorePath & path)
-{
- Worker worker(*this);
- GoalPtr goal = worker.makeSubstitutionGoal(path, Repair);
- Goals goals = {goal};
-
- worker.run(goals);
-
- if (goal->exitCode != Goal::ecSuccess) {
- /* Since substituting the path didn't work, if we have a valid
- deriver, then rebuild the deriver. */
- auto info = queryPathInfo(path);
- if (info->deriver && isValidPath(*info->deriver)) {
- goals.clear();
- goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair));
- worker.run(goals);
- } else
- throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path));
- }
-}
-
-
}
diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh
new file mode 100644
index 000000000..4976207e0
--- /dev/null
+++ b/src/libstore/build/derivation-goal.hh
@@ -0,0 +1,379 @@
+#pragma once
+
+#include "parsed-derivations.hh"
+#include "lock.hh"
+#include "local-store.hh"
+#include "goal.hh"
+
+namespace nix {
+
+using std::map;
+
+struct HookInstance;
+
+typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
+
+/* Unless we are repairing, we don't both to test validity and just assume it,
+ so the choices are `Absent` or `Valid`. */
+enum struct PathStatus {
+ Corrupt,
+ Absent,
+ Valid,
+};
+
+struct InitialOutputStatus {
+ StorePath path;
+ PathStatus status;
+ /* Valid in the store, and additionally non-corrupt if we are repairing */
+ bool isValid() const {
+ return status == PathStatus::Valid;
+ }
+ /* Merely present, allowed to be corrupt */
+ bool isPresent() const {
+ return status == PathStatus::Corrupt
+ || status == PathStatus::Valid;
+ }
+};
+
+struct InitialOutput {
+ bool wanted;
+ std::optional<InitialOutputStatus> known;
+};
+
+class DerivationGoal : public Goal
+{
+private:
+ /* Whether to use an on-disk .drv file. */
+ bool useDerivation;
+
+ /* The path of the derivation. */
+ StorePath drvPath;
+
+ /* The specific outputs that we need to build. Empty means all of
+ them. */
+ StringSet wantedOutputs;
+
+ /* Whether additional wanted outputs have been added. */
+ bool needRestart = false;
+
+ /* Whether to retry substituting the outputs after building the
+ inputs. */
+ bool retrySubstitution;
+
+ /* The derivation stored at drvPath. */
+ std::unique_ptr<BasicDerivation> drv;
+
+ std::unique_ptr<ParsedDerivation> parsedDrv;
+
+ /* The remainder is state held during the build. */
+
+ /* Locks on (fixed) output paths. */
+ PathLocks outputLocks;
+
+ /* All input paths (that is, the union of FS closures of the
+ immediate input paths). */
+ StorePathSet inputPaths;
+
+ std::map<std::string, InitialOutput> initialOutputs;
+
+ /* User selected for running the builder. */
+ std::unique_ptr<UserLock> buildUser;
+
+ /* The process ID of the builder. */
+ Pid pid;
+
+ /* The temporary directory. */
+ Path tmpDir;
+
+ /* The path of the temporary directory in the sandbox. */
+ Path tmpDirInSandbox;
+
+ /* File descriptor for the log file. */
+ AutoCloseFD fdLogFile;
+ std::shared_ptr<BufferedSink> logFileSink, logSink;
+
+ /* Number of bytes received from the builder's stdout/stderr. */
+ unsigned long logSize;
+
+ /* The most recent log lines. */
+ std::list<std::string> logTail;
+
+ std::string currentLogLine;
+ size_t currentLogLinePos = 0; // to handle carriage return
+
+ std::string currentHookLine;
+
+ /* Pipe for the builder's standard output/error. */
+ Pipe builderOut;
+
+ /* Pipe for synchronising updates to the builder namespaces. */
+ Pipe userNamespaceSync;
+
+ /* The mount namespace of the builder, used to add additional
+ paths to the sandbox as a result of recursive Nix calls. */
+ AutoCloseFD sandboxMountNamespace;
+
+ /* On Linux, whether we're doing the build in its own user
+ namespace. */
+ bool usingUserNamespace = true;
+
+ /* The build hook. */
+ std::unique_ptr<HookInstance> hook;
+
+ /* Whether we're currently doing a chroot build. */
+ bool useChroot = false;
+
+ Path chrootRootDir;
+
+ /* RAII object to delete the chroot directory. */
+ std::shared_ptr<AutoDelete> autoDelChroot;
+
+ /* The sort of derivation we are building. */
+ DerivationType derivationType;
+
+ /* Whether to run the build in a private network namespace. */
+ bool privateNetwork = false;
+
+ typedef void (DerivationGoal::*GoalState)();
+ GoalState state;
+
+ /* Stuff we need to pass to initChild(). */
+ struct ChrootPath {
+ Path source;
+ bool optional;
+ ChrootPath(Path source = "", bool optional = false)
+ : source(source), optional(optional)
+ { }
+ };
+ typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
+ DirsInChroot dirsInChroot;
+
+ typedef map<string, string> Environment;
+ Environment env;
+
+#if __APPLE__
+ typedef string SandboxProfile;
+ SandboxProfile additionalSandboxProfile;
+#endif
+
+ /* Hash rewriting. */
+ StringMap inputRewrites, outputRewrites;
+ typedef map<StorePath, StorePath> RedirectedOutputs;
+ RedirectedOutputs redirectedOutputs;
+
+ /* The outputs paths used during the build.
+
+ - Input-addressed derivations or fixed content-addressed outputs are
+ sometimes built when some of their outputs already exist, and can not
+ be hidden via sandboxing. We use temporary locations instead and
+ rewrite after the build. Otherwise the regular predetermined paths are
+ put here.
+
+ - Floating content-addressed derivations do not know their final build
+ output paths until the outputs are hashed, so random locations are
+ used, and then renamed. The randomness helps guard against hidden
+ self-references.
+ */
+ OutputPathMap scratchOutputs;
+
+ /* The final output paths of the build.
+
+ - For input-addressed derivations, always the precomputed paths
+
+ - For content-addressed derivations, calcuated from whatever the hash
+ ends up being. (Note that fixed outputs derivations that produce the
+ "wrong" output still install that data under its true content-address.)
+ */
+ OutputPathMap finalOutputs;
+
+ BuildMode buildMode;
+
+ /* If we're repairing without a chroot, there may be outputs that
+ are valid but corrupt. So we redirect these outputs to
+ temporary paths. */
+ StorePathSet redirectedBadOutputs;
+
+ BuildResult result;
+
+ /* The current round, if we're building multiple times. */
+ size_t curRound = 1;
+
+ size_t nrRounds;
+
+ /* Path registration info from the previous round, if we're
+ building multiple times. Since this contains the hash, it
+ allows us to compare whether two rounds produced the same
+ result. */
+ std::map<Path, ValidPathInfo> prevInfos;
+
+ uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
+ gid_t sandboxGid() { return usingUserNamespace ? 100 : buildUser->getGID(); }
+
+ const static Path homeDir;
+
+ std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
+
+ std::unique_ptr<Activity> act;
+
+ /* Activity that denotes waiting for a lock. */
+ std::unique_ptr<Activity> actLock;
+
+ std::map<ActivityId, Activity> builderActivities;
+
+ /* The remote machine on which we're building. */
+ std::string machineName;
+
+ /* The recursive Nix daemon socket. */
+ AutoCloseFD daemonSocket;
+
+ /* The daemon main thread. */
+ std::thread daemonThread;
+
+ /* The daemon worker threads. */
+ std::vector<std::thread> daemonWorkerThreads;
+
+ /* Paths that were added via recursive Nix calls. */
+ StorePathSet addedPaths;
+
+ /* Recursive Nix calls are only allowed to build or realize paths
+ in the original input closure or added via a recursive Nix call
+ (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
+ /nix/store/<bla> is some arbitrary path in a binary cache). */
+ bool isAllowed(const StorePath & path)
+ {
+ return inputPaths.count(path) || addedPaths.count(path);
+ }
+
+ friend struct RestrictedStore;
+
+public:
+ DerivationGoal(const StorePath & drvPath,
+ const StringSet & wantedOutputs, Worker & worker,
+ BuildMode buildMode = bmNormal);
+ DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
+ const StringSet & wantedOutputs, Worker & worker,
+ BuildMode buildMode = bmNormal);
+ ~DerivationGoal();
+
+ /* Whether we need to perform hash rewriting if there are valid output paths. */
+ bool needsHashRewrite();
+
+ void timedOut(Error && ex) override;
+
+ string key() override;
+
+ void work() override;
+
+ StorePath getDrvPath()
+ {
+ return drvPath;
+ }
+
+ /* Add wanted outputs to an already existing derivation goal. */
+ void addWantedOutputs(const StringSet & outputs);
+
+ BuildResult getResult() { return result; }
+
+private:
+ /* The states. */
+ void getDerivation();
+ void loadDerivation();
+ void haveDerivation();
+ void outputsSubstitutionTried();
+ void gaveUpOnSubstitution();
+ void closureRepaired();
+ void inputsRealised();
+ void tryToBuild();
+ void tryLocalBuild();
+ void buildDone();
+
+ void resolvedFinished();
+
+ /* Is the build hook willing to perform the build? */
+ HookReply tryBuildHook();
+
+ /* Start building a derivation. */
+ void startBuilder();
+
+ /* Fill in the environment for the builder. */
+ void initEnv();
+
+ /* Setup tmp dir location. */
+ void initTmpDir();
+
+ /* Write a JSON file containing the derivation attributes. */
+ void writeStructuredAttrs();
+
+ void startDaemon();
+
+ void stopDaemon();
+
+ /* Add 'path' to the set of paths that may be referenced by the
+ outputs, and make it appear in the sandbox. */
+ void addDependency(const StorePath & path);
+
+ /* Make a file owned by the builder. */
+ void chownToBuilder(const Path & path);
+
+ /* Run the builder's process. */
+ void runChild();
+
+ friend int childEntry(void *);
+
+ /* Check that the derivation outputs all exist and register them
+ as valid. */
+ void registerOutputs();
+
+ /* Check that an output meets the requirements specified by the
+ 'outputChecks' attribute (or the legacy
+ '{allowed,disallowed}{References,Requisites}' attributes). */
+ void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
+
+ /* Open a log file and a pipe to it. */
+ Path openLogFile();
+
+ /* Close the log file. */
+ void closeLogFile();
+
+ /* Delete the temporary directory, if we have one. */
+ void deleteTmpDir(bool force);
+
+ /* Callback used by the worker to write to the log. */
+ void handleChildOutput(int fd, const string & data) override;
+ void handleEOF(int fd) override;
+ void flushLine();
+
+ /* Wrappers around the corresponding Store methods that first consult the
+ derivation. This is currently needed because when there is no drv file
+ there also is no DB entry. */
+ std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
+ OutputPathMap queryDerivationOutputMap();
+
+ /* Return the set of (in)valid paths. */
+ void checkPathValidity();
+
+ /* Forcibly kill the child process, if any. */
+ void killChild();
+
+ /* Create alternative path calculated from but distinct from the
+ input, so we can avoid overwriting outputs (or other store paths)
+ that already exist. */
+ StorePath makeFallbackPath(const StorePath & path);
+ /* Make a path to another based on the output name along with the
+ derivation hash. */
+ /* FIXME add option to randomize, so we can audit whether our
+ rewrites caught everything */
+ StorePath makeFallbackPath(std::string_view outputName);
+
+ void repairClosure();
+
+ void started();
+
+ void done(
+ BuildResult::Status status,
+ std::optional<Error> ex = {});
+
+ StorePathSet exportReferences(const StorePathSet & storePaths);
+};
+
+}
diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc
new file mode 100644
index 000000000..2dd7a4d37
--- /dev/null
+++ b/src/libstore/build/goal.cc
@@ -0,0 +1,89 @@
+#include "goal.hh"
+#include "worker.hh"
+
+namespace nix {
+
+
+bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
+ string s1 = a->key();
+ string s2 = b->key();
+ return s1 < s2;
+}
+
+
+void addToWeakGoals(WeakGoals & goals, GoalPtr p)
+{
+ // FIXME: necessary?
+ // FIXME: O(n)
+ for (auto & i : goals)
+ if (i.lock() == p) return;
+ goals.push_back(p);
+}
+
+
+void Goal::addWaitee(GoalPtr waitee)
+{
+ waitees.insert(waitee);
+ addToWeakGoals(waitee->waiters, shared_from_this());
+}
+
+
+void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
+{
+ assert(waitees.find(waitee) != waitees.end());
+ waitees.erase(waitee);
+
+ trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size()));
+
+ if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed;
+
+ if (result == ecNoSubstituters) ++nrNoSubstituters;
+
+ if (result == ecIncompleteClosure) ++nrIncompleteClosure;
+
+ if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) {
+
+ /* If we failed and keepGoing is not set, we remove all
+ remaining waitees. */
+ for (auto & goal : waitees) {
+ WeakGoals waiters2;
+ for (auto & j : goal->waiters)
+ if (j.lock() != shared_from_this()) waiters2.push_back(j);
+ goal->waiters = waiters2;
+ }
+ waitees.clear();
+
+ worker.wakeUp(shared_from_this());
+ }
+}
+
+
+void Goal::amDone(ExitCode result, std::optional<Error> ex)
+{
+ trace("done");
+ assert(exitCode == ecBusy);
+ assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure);
+ exitCode = result;
+
+ if (ex) {
+ if (!waiters.empty())
+ logError(ex->info());
+ else
+ this->ex = std::move(*ex);
+ }
+
+ for (auto & i : waiters) {
+ GoalPtr goal = i.lock();
+ if (goal) goal->waiteeDone(shared_from_this(), result);
+ }
+ waiters.clear();
+ worker.removeGoal(shared_from_this());
+}
+
+
+void Goal::trace(const FormatOrString & fs)
+{
+ debug("%1%: %2%", name, fs.s);
+}
+
+}
diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh
new file mode 100644
index 000000000..360c160ce
--- /dev/null
+++ b/src/libstore/build/goal.hh
@@ -0,0 +1,107 @@
+#pragma once
+
+#include "types.hh"
+#include "store-api.hh"
+
+namespace nix {
+
+/* Forward definition. */
+struct Goal;
+struct Worker;
+
+/* A pointer to a goal. */
+typedef std::shared_ptr<Goal> GoalPtr;
+typedef std::weak_ptr<Goal> WeakGoalPtr;
+
+struct CompareGoalPtrs {
+ bool operator() (const GoalPtr & a, const GoalPtr & b) const;
+};
+
+/* Set of goals. */
+typedef set<GoalPtr, CompareGoalPtrs> Goals;
+typedef list<WeakGoalPtr> WeakGoals;
+
+/* A map of paths to goals (and the other way around). */
+typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
+
+struct Goal : public std::enable_shared_from_this<Goal>
+{
+ typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
+
+ /* Backlink to the worker. */
+ Worker & worker;
+
+ /* Goals that this goal is waiting for. */
+ Goals waitees;
+
+ /* Goals waiting for this one to finish. Must use weak pointers
+ here to prevent cycles. */
+ WeakGoals waiters;
+
+ /* Number of goals we are/were waiting for that have failed. */
+ unsigned int nrFailed;
+
+ /* Number of substitution goals we are/were waiting for that
+ failed because there are no substituters. */
+ unsigned int nrNoSubstituters;
+
+ /* Number of substitution goals we are/were waiting for that
+ failed because othey had unsubstitutable references. */
+ unsigned int nrIncompleteClosure;
+
+ /* Name of this goal for debugging purposes. */
+ string name;
+
+ /* Whether the goal is finished. */
+ ExitCode exitCode;
+
+ /* Exception containing an error message, if any. */
+ std::optional<Error> ex;
+
+ Goal(Worker & worker) : worker(worker)
+ {
+ nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
+ exitCode = ecBusy;
+ }
+
+ virtual ~Goal()
+ {
+ trace("goal destroyed");
+ }
+
+ virtual void work() = 0;
+
+ void addWaitee(GoalPtr waitee);
+
+ virtual void waiteeDone(GoalPtr waitee, ExitCode result);
+
+ virtual void handleChildOutput(int fd, const string & data)
+ {
+ abort();
+ }
+
+ virtual void handleEOF(int fd)
+ {
+ abort();
+ }
+
+ void trace(const FormatOrString & fs);
+
+ string getName()
+ {
+ return name;
+ }
+
+ /* Callback in case of a timeout. It should wake up its waiters,
+ get rid of any running child processes that are being monitored
+ by the worker (important!), etc. */
+ virtual void timedOut(Error && ex) = 0;
+
+ virtual string key() = 0;
+
+ void amDone(ExitCode result, std::optional<Error> ex = {});
+};
+
+void addToWeakGoals(WeakGoals & goals, GoalPtr p);
+
+}
diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc
new file mode 100644
index 000000000..0f6f580be
--- /dev/null
+++ b/src/libstore/build/hook-instance.cc
@@ -0,0 +1,72 @@
+#include "globals.hh"
+#include "hook-instance.hh"
+
+namespace nix {
+
+HookInstance::HookInstance()
+{
+ debug("starting build hook '%s'", settings.buildHook);
+
+ /* Create a pipe to get the output of the child. */
+ fromHook.create();
+
+ /* Create the communication pipes. */
+ toHook.create();
+
+ /* Create a pipe to get the output of the builder. */
+ builderOut.create();
+
+ /* Fork the hook. */
+ pid = startProcess([&]() {
+
+ commonChildInit(fromHook);
+
+ if (chdir("/") == -1) throw SysError("changing into /");
+
+ /* Dup the communication pipes. */
+ if (dup2(toHook.readSide.get(), STDIN_FILENO) == -1)
+ throw SysError("dupping to-hook read side");
+
+ /* Use fd 4 for the builder's stdout/stderr. */
+ if (dup2(builderOut.writeSide.get(), 4) == -1)
+ throw SysError("dupping builder's stdout/stderr");
+
+ /* Hack: pass the read side of that fd to allow build-remote
+ to read SSH error messages. */
+ if (dup2(builderOut.readSide.get(), 5) == -1)
+ throw SysError("dupping builder's stdout/stderr");
+
+ Strings args = {
+ std::string(baseNameOf(settings.buildHook.get())),
+ std::to_string(verbosity),
+ };
+
+ execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data());
+
+ throw SysError("executing '%s'", settings.buildHook);
+ });
+
+ pid.setSeparatePG(true);
+ fromHook.writeSide = -1;
+ toHook.readSide = -1;
+
+ sink = FdSink(toHook.writeSide.get());
+ std::map<std::string, Config::SettingInfo> settings;
+ globalConfig.getSettings(settings);
+ for (auto & setting : settings)
+ sink << 1 << setting.first << setting.second.value;
+ sink << 0;
+}
+
+
+HookInstance::~HookInstance()
+{
+ try {
+ toHook.writeSide = -1;
+ if (pid != -1) pid.kill();
+ } catch (...) {
+ ignoreException();
+ }
+}
+
+}
diff --git a/src/libstore/build/hook-instance.hh b/src/libstore/build/hook-instance.hh
new file mode 100644
index 000000000..9e8cff128
--- /dev/null
+++ b/src/libstore/build/hook-instance.hh
@@ -0,0 +1,31 @@
+#pragma once
+
+#include "logging.hh"
+#include "serialise.hh"
+
+namespace nix {
+
+struct HookInstance
+{
+ /* Pipes for talking to the build hook. */
+ Pipe toHook;
+
+ /* Pipe for the hook's standard output/error. */
+ Pipe fromHook;
+
+ /* Pipe for the builder's standard output/error. */
+ Pipe builderOut;
+
+ /* The process ID of the hook. */
+ Pid pid;
+
+ FdSink sink;
+
+ std::map<ActivityId, Activity> activities;
+
+ HookInstance();
+
+ ~HookInstance();
+};
+
+}
diff --git a/src/libstore/build/local-store-build.cc b/src/libstore/build/local-store-build.cc
new file mode 100644
index 000000000..a05fb5805
--- /dev/null
+++ b/src/libstore/build/local-store-build.cc
@@ -0,0 +1,126 @@
+#include "machines.hh"
+#include "worker.hh"
+#include "substitution-goal.hh"
+#include "derivation-goal.hh"
+
+namespace nix {
+
+static void primeCache(Store & store, const std::vector<StorePathWithOutputs> & paths)
+{
+ StorePathSet willBuild, willSubstitute, unknown;
+ uint64_t downloadSize, narSize;
+ store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
+
+ if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty())
+ throw Error(
+ "%d derivations need to be built, but neither local builds ('--max-jobs') "
+ "nor remote builds ('--builders') are enabled", willBuild.size());
+}
+
+
+void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
+{
+ Worker worker(*this);
+
+ primeCache(*this, drvPaths);
+
+ Goals goals;
+ for (auto & path : drvPaths) {
+ if (path.path.isDerivation())
+ goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode));
+ else
+ goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair));
+ }
+
+ worker.run(goals);
+
+ StorePathSet failed;
+ std::optional<Error> ex;
+ for (auto & i : goals) {
+ if (i->ex) {
+ if (ex)
+ logError(i->ex->info());
+ else
+ ex = i->ex;
+ }
+ if (i->exitCode != Goal::ecSuccess) {
+ DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get());
+ if (i2) failed.insert(i2->getDrvPath());
+ else failed.insert(dynamic_cast<SubstitutionGoal *>(i.get())->getStorePath());
+ }
+ }
+
+ if (failed.size() == 1 && ex) {
+ ex->status = worker.exitStatus();
+ throw *ex;
+ } else if (!failed.empty()) {
+ if (ex) logError(ex->info());
+ throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed));
+ }
+}
+
+BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
+ BuildMode buildMode)
+{
+ Worker worker(*this);
+ auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode);
+
+ BuildResult result;
+
+ try {
+ worker.run(Goals{goal});
+ result = goal->getResult();
+ } catch (Error & e) {
+ result.status = BuildResult::MiscFailure;
+ result.errorMsg = e.msg();
+ }
+
+ return result;
+}
+
+
+void LocalStore::ensurePath(const StorePath & path)
+{
+ /* If the path is already valid, we're done. */
+ if (isValidPath(path)) return;
+
+ primeCache(*this, {{path}});
+
+ Worker worker(*this);
+ GoalPtr goal = worker.makeSubstitutionGoal(path);
+ Goals goals = {goal};
+
+ worker.run(goals);
+
+ if (goal->exitCode != Goal::ecSuccess) {
+ if (goal->ex) {
+ goal->ex->status = worker.exitStatus();
+ throw *goal->ex;
+ } else
+ throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path));
+ }
+}
+
+
+void LocalStore::repairPath(const StorePath & path)
+{
+ Worker worker(*this);
+ GoalPtr goal = worker.makeSubstitutionGoal(path, Repair);
+ Goals goals = {goal};
+
+ worker.run(goals);
+
+ if (goal->exitCode != Goal::ecSuccess) {
+ /* Since substituting the path didn't work, if we have a valid
+ deriver, then rebuild the deriver. */
+ auto info = queryPathInfo(path);
+ if (info->deriver && isValidPath(*info->deriver)) {
+ goals.clear();
+ goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair));
+ worker.run(goals);
+ } else
+ throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path));
+ }
+}
+
+}
diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc
new file mode 100644
index 000000000..d16584f65
--- /dev/null
+++ b/src/libstore/build/substitution-goal.cc
@@ -0,0 +1,296 @@
+#include "worker.hh"
+#include "substitution-goal.hh"
+#include "nar-info.hh"
+#include "finally.hh"
+
+namespace nix {
+
+SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
+ : Goal(worker)
+ , storePath(storePath)
+ , repair(repair)
+ , ca(ca)
+{
+ state = &SubstitutionGoal::init;
+ name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
+ trace("created");
+ maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions);
+}
+
+
+SubstitutionGoal::~SubstitutionGoal()
+{
+ try {
+ if (thr.joinable()) {
+ // FIXME: signal worker thread to quit.
+ thr.join();
+ worker.childTerminated(this);
+ }
+ } catch (...) {
+ ignoreException();
+ }
+}
+
+
+void SubstitutionGoal::work()
+{
+ (this->*state)();
+}
+
+
+void SubstitutionGoal::init()
+{
+ trace("init");
+
+ worker.store.addTempRoot(storePath);
+
+ /* If the path already exists we're done. */
+ if (!repair && worker.store.isValidPath(storePath)) {
+ amDone(ecSuccess);
+ return;
+ }
+
+ if (settings.readOnlyMode)
+ throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath));
+
+ subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
+
+ tryNext();
+}
+
+
+void SubstitutionGoal::tryNext()
+{
+ trace("trying next substituter");
+
+ if (subs.size() == 0) {
+ /* None left. Terminate this goal and let someone else deal
+ with it. */
+ debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath));
+
+ /* Hack: don't indicate failure if there were no substituters.
+ In that case the calling derivation should just do a
+ build. */
+ amDone(substituterFailed ? ecFailed : ecNoSubstituters);
+
+ if (substituterFailed) {
+ worker.failedSubstitutions++;
+ worker.updateProgress();
+ }
+
+ return;
+ }
+
+ sub = subs.front();
+ subs.pop_front();
+
+ if (ca) {
+ subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
+ if (sub->storeDir == worker.store.storeDir)
+ assert(subPath == storePath);
+ } else if (sub->storeDir != worker.store.storeDir) {
+ tryNext();
+ return;
+ }
+
+ try {
+ // FIXME: make async
+ info = sub->queryPathInfo(subPath ? *subPath : storePath);
+ } catch (InvalidPath &) {
+ tryNext();
+ return;
+ } catch (SubstituterDisabled &) {
+ if (settings.tryFallback) {
+ tryNext();
+ return;
+ }
+ throw;
+ } catch (Error & e) {
+ if (settings.tryFallback) {
+ logError(e.info());
+ tryNext();
+ return;
+ }
+ throw;
+ }
+
+ if (info->path != storePath) {
+ if (info->isContentAddressed(*sub) && info->references.empty()) {
+ auto info2 = std::make_shared<ValidPathInfo>(*info);
+ info2->path = storePath;
+ info = info2;
+ } else {
+ printError("asked '%s' for '%s' but got '%s'",
+ sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path));
+ tryNext();
+ return;
+ }
+ }
+
+ /* Update the total expected download size. */
+ auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
+
+ maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, info->narSize);
+
+ maintainExpectedDownload =
+ narInfo && narInfo->fileSize
+ ? std::make_unique<MaintainCount<uint64_t>>(worker.expectedDownloadSize, narInfo->fileSize)
+ : nullptr;
+
+ worker.updateProgress();
+
+ /* Bail out early if this substituter lacks a valid
+ signature. LocalStore::addToStore() also checks for this, but
+ only after we've downloaded the path. */
+ if (worker.store.requireSigs
+ && !sub->isTrusted
+ && !info->checkSignatures(worker.store, worker.store.getPublicKeys()))
+ {
+ logWarning({
+ .name = "Invalid path signature",
+ .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'",
+ sub->getUri(), worker.store.printStorePath(storePath))
+ });
+ tryNext();
+ return;
+ }
+
+ /* To maintain the closure invariant, we first have to realise the
+ paths referenced by this one. */
+ for (auto & i : info->references)
+ if (i != storePath) /* ignore self-references */
+ addWaitee(worker.makeSubstitutionGoal(i));
+
+ if (waitees.empty()) /* to prevent hang (no wake-up event) */
+ referencesValid();
+ else
+ state = &SubstitutionGoal::referencesValid;
+}
+
+
+void SubstitutionGoal::referencesValid()
+{
+ trace("all references realised");
+
+ if (nrFailed > 0) {
+ debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath));
+ amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
+ return;
+ }
+
+ for (auto & i : info->references)
+ if (i != storePath) /* ignore self-references */
+ assert(worker.store.isValidPath(i));
+
+ state = &SubstitutionGoal::tryToRun;
+ worker.wakeUp(shared_from_this());
+}
+
+
+void SubstitutionGoal::tryToRun()
+{
+ trace("trying to run");
+
+ /* Make sure that we are allowed to start a build. Note that even
+ if maxBuildJobs == 0 (no local builds allowed), we still allow
+ a substituter to run. This is because substitutions cannot be
+ distributed to another machine via the build hook. */
+ if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) {
+ worker.waitForBuildSlot(shared_from_this());
+ return;
+ }
+
+ maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
+ worker.updateProgress();
+
+ outPipe.create();
+
+ promise = std::promise<void>();
+
+ thr = std::thread([this]() {
+ try {
+ /* Wake up the worker loop when we're done. */
+ Finally updateStats([this]() { outPipe.writeSide = -1; });
+
+ Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()});
+ PushActivity pact(act.id);
+
+ copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()),
+ subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
+
+ promise.set_value();
+ } catch (...) {
+ promise.set_exception(std::current_exception());
+ }
+ });
+
+ worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false);
+
+ state = &SubstitutionGoal::finished;
+}
+
+
+void SubstitutionGoal::finished()
+{
+ trace("substitute finished");
+
+ thr.join();
+ worker.childTerminated(this);
+
+ try {
+ promise.get_future().get();
+ } catch (std::exception & e) {
+ printError(e.what());
+
+ /* Cause the parent build to fail unless --fallback is given,
+ or the substitute has disappeared. The latter case behaves
+ the same as the substitute never having existed in the
+ first place. */
+ try {
+ throw;
+ } catch (SubstituteGone &) {
+ } catch (...) {
+ substituterFailed = true;
+ }
+
+ /* Try the next substitute. */
+ state = &SubstitutionGoal::tryNext;
+ worker.wakeUp(shared_from_this());
+ return;
+ }
+
+ worker.markContentsGood(storePath);
+
+ printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath));
+
+ maintainRunningSubstitutions.reset();
+
+ maintainExpectedSubstitutions.reset();
+ worker.doneSubstitutions++;
+
+ if (maintainExpectedDownload) {
+ auto fileSize = maintainExpectedDownload->delta;
+ maintainExpectedDownload.reset();
+ worker.doneDownloadSize += fileSize;
+ }
+
+ worker.doneNarSize += maintainExpectedNar->delta;
+ maintainExpectedNar.reset();
+
+ worker.updateProgress();
+
+ amDone(ecSuccess);
+}
+
+
+void SubstitutionGoal::handleChildOutput(int fd, const string & data)
+{
+}
+
+
+void SubstitutionGoal::handleEOF(int fd)
+{
+ if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
+}
+
+}
diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh
new file mode 100644
index 000000000..3ae9a9e6b
--- /dev/null
+++ b/src/libstore/build/substitution-goal.hh
@@ -0,0 +1,89 @@
+#pragma once
+
+#include "lock.hh"
+#include "store-api.hh"
+#include "goal.hh"
+
+namespace nix {
+
+class Worker;
+
+class SubstitutionGoal : public Goal
+{
+ friend class Worker;
+
+private:
+ /* The store path that should be realised through a substitute. */
+ StorePath storePath;
+
+ /* The path the substituter refers to the path as. This will be
+ * different when the stores have different names. */
+ std::optional<StorePath> subPath;
+
+ /* The remaining substituters. */
+ std::list<ref<Store>> subs;
+
+ /* The current substituter. */
+ std::shared_ptr<Store> sub;
+
+ /* Whether a substituter failed. */
+ bool substituterFailed = false;
+
+ /* Path info returned by the substituter's query info operation. */
+ std::shared_ptr<const ValidPathInfo> info;
+
+ /* Pipe for the substituter's standard output. */
+ Pipe outPipe;
+
+ /* The substituter thread. */
+ std::thread thr;
+
+ std::promise<void> promise;
+
+ /* Whether to try to repair a valid path. */
+ RepairFlag repair;
+
+ /* Location where we're downloading the substitute. Differs from
+ storePath when doing a repair. */
+ Path destPath;
+
+ std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
+ maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
+
+ typedef void (SubstitutionGoal::*GoalState)();
+ GoalState state;
+
+ /* Content address for recomputing store path */
+ std::optional<ContentAddress> ca;
+
+public:
+ SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
+ ~SubstitutionGoal();
+
+ void timedOut(Error && ex) override { abort(); };
+
+ string key() override
+ {
+ /* "a$" ensures substitution goals happen before derivation
+ goals. */
+ return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
+ }
+
+ void work() override;
+
+ /* The states. */
+ void init();
+ void tryNext();
+ void gotInfo();
+ void referencesValid();
+ void tryToRun();
+ void finished();
+
+ /* Callback used by the worker to write to the log. */
+ void handleChildOutput(int fd, const string & data) override;
+ void handleEOF(int fd) override;
+
+ StorePath getStorePath() { return storePath; }
+};
+
+}
diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
new file mode 100644
index 000000000..5c3fe2f57
--- /dev/null
+++ b/src/libstore/build/worker.cc
@@ -0,0 +1,455 @@
+#include "machines.hh"
+#include "worker.hh"
+#include "substitution-goal.hh"
+#include "derivation-goal.hh"
+#include "hook-instance.hh"
+
+#include <poll.h>
+
+namespace nix {
+
+Worker::Worker(LocalStore & store)
+ : act(*logger, actRealise)
+ , actDerivations(*logger, actBuilds)
+ , actSubstitutions(*logger, actCopyPaths)
+ , store(store)
+{
+ /* Debugging: prevent recursive workers. */
+ nrLocalBuilds = 0;
+ lastWokenUp = steady_time_point::min();
+ permanentFailure = false;
+ timedOut = false;
+ hashMismatch = false;
+ checkMismatch = false;
+}
+
+
+Worker::~Worker()
+{
+ /* Explicitly get rid of all strong pointers now. After this all
+ goals that refer to this worker should be gone. (Otherwise we
+ are in trouble, since goals may call childTerminated() etc. in
+ their destructors). */
+ topGoals.clear();
+
+ assert(expectedSubstitutions == 0);
+ assert(expectedDownloadSize == 0);
+ assert(expectedNarSize == 0);
+}
+
+
+std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
+ const StorePath & drvPath,
+ const StringSet & wantedOutputs,
+ std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal)
+{
+ WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath];
+ GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME
+ std::shared_ptr<DerivationGoal> goal;
+ if (!abstract_goal) {
+ goal = mkDrvGoal();
+ abstract_goal_weak = goal;
+ wakeUp(goal);
+ } else {
+ goal = std::dynamic_pointer_cast<DerivationGoal>(abstract_goal);
+ assert(goal);
+ goal->addWantedOutputs(wantedOutputs);
+ }
+ return goal;
+}
+
+
+std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
+ const StringSet & wantedOutputs, BuildMode buildMode)
+{
+ return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
+ return std::make_shared<DerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
+ });
+}
+
+
+std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
+ const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode)
+{
+ return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
+ return std::make_shared<DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
+ });
+}
+
+
+GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
+{
+ WeakGoalPtr & goal_weak = substitutionGoals[path];
+ GoalPtr goal = goal_weak.lock(); // FIXME
+ if (!goal) {
+ goal = std::make_shared<SubstitutionGoal>(path, *this, repair, ca);
+ goal_weak = goal;
+ wakeUp(goal);
+ }
+ return goal;
+}
+
+
+static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap)
+{
+ /* !!! inefficient */
+ for (WeakGoalMap::iterator i = goalMap.begin();
+ i != goalMap.end(); )
+ if (i->second.lock() == goal) {
+ WeakGoalMap::iterator j = i; ++j;
+ goalMap.erase(i);
+ i = j;
+ }
+ else ++i;
+}
+
+
+void Worker::removeGoal(GoalPtr goal)
+{
+ nix::removeGoal(goal, derivationGoals);
+ nix::removeGoal(goal, substitutionGoals);
+ if (topGoals.find(goal) != topGoals.end()) {
+ topGoals.erase(goal);
+ /* If a top-level goal failed, then kill all other goals
+ (unless keepGoing was set). */
+ if (goal->exitCode == Goal::ecFailed && !settings.keepGoing)
+ topGoals.clear();
+ }
+
+ /* Wake up goals waiting for any goal to finish. */
+ for (auto & i : waitingForAnyGoal) {
+ GoalPtr goal = i.lock();
+ if (goal) wakeUp(goal);
+ }
+
+ waitingForAnyGoal.clear();
+}
+
+
+void Worker::wakeUp(GoalPtr goal)
+{
+ goal->trace("woken up");
+ addToWeakGoals(awake, goal);
+}
+
+
+unsigned Worker::getNrLocalBuilds()
+{
+ return nrLocalBuilds;
+}
+
+
+void Worker::childStarted(GoalPtr goal, const set<int> & fds,
+ bool inBuildSlot, bool respectTimeouts)
+{
+ Child child;
+ child.goal = goal;
+ child.goal2 = goal.get();
+ child.fds = fds;
+ child.timeStarted = child.lastOutput = steady_time_point::clock::now();
+ child.inBuildSlot = inBuildSlot;
+ child.respectTimeouts = respectTimeouts;
+ children.emplace_back(child);
+ if (inBuildSlot) nrLocalBuilds++;
+}
+
+
+void Worker::childTerminated(Goal * goal, bool wakeSleepers)
+{
+ auto i = std::find_if(children.begin(), children.end(),
+ [&](const Child & child) { return child.goal2 == goal; });
+ if (i == children.end()) return;
+
+ if (i->inBuildSlot) {
+ assert(nrLocalBuilds > 0);
+ nrLocalBuilds--;
+ }
+
+ children.erase(i);
+
+ if (wakeSleepers) {
+
+ /* Wake up goals waiting for a build slot. */
+ for (auto & j : wantingToBuild) {
+ GoalPtr goal = j.lock();
+ if (goal) wakeUp(goal);
+ }
+
+ wantingToBuild.clear();
+ }
+}
+
+
+void Worker::waitForBuildSlot(GoalPtr goal)
+{
+ debug("wait for build slot");
+ if (getNrLocalBuilds() < settings.maxBuildJobs)
+ wakeUp(goal); /* we can do it right away */
+ else
+ addToWeakGoals(wantingToBuild, goal);
+}
+
+
+void Worker::waitForAnyGoal(GoalPtr goal)
+{
+ debug("wait for any goal");
+ addToWeakGoals(waitingForAnyGoal, goal);
+}
+
+
+void Worker::waitForAWhile(GoalPtr goal)
+{
+ debug("wait for a while");
+ addToWeakGoals(waitingForAWhile, goal);
+}
+
+
+void Worker::run(const Goals & _topGoals)
+{
+ for (auto & i : _topGoals) topGoals.insert(i);
+
+ debug("entered goal loop");
+
+ while (1) {
+
+ checkInterrupt();
+
+ store.autoGC(false);
+
+ /* Call every wake goal (in the ordering established by
+ CompareGoalPtrs). */
+ while (!awake.empty() && !topGoals.empty()) {
+ Goals awake2;
+ for (auto & i : awake) {
+ GoalPtr goal = i.lock();
+ if (goal) awake2.insert(goal);
+ }
+ awake.clear();
+ for (auto & goal : awake2) {
+ checkInterrupt();
+ goal->work();
+ if (topGoals.empty()) break; // stuff may have been cancelled
+ }
+ }
+
+ if (topGoals.empty()) break;
+
+ /* Wait for input. */
+ if (!children.empty() || !waitingForAWhile.empty())
+ waitForInput();
+ else {
+ if (awake.empty() && 0 == settings.maxBuildJobs)
+ {
+ if (getMachines().empty())
+ throw Error("unable to start any build; either increase '--max-jobs' "
+ "or enable remote builds."
+ "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
+ else
+ throw Error("unable to start any build; remote machines may not have "
+ "all required system features."
+ "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
+
+ }
+ assert(!awake.empty());
+ }
+ }
+
+ /* If --keep-going is not set, it's possible that the main goal
+ exited while some of its subgoals were still active. But if
+ --keep-going *is* set, then they must all be finished now. */
+ assert(!settings.keepGoing || awake.empty());
+ assert(!settings.keepGoing || wantingToBuild.empty());
+ assert(!settings.keepGoing || children.empty());
+}
+
+void Worker::waitForInput()
+{
+ printMsg(lvlVomit, "waiting for children");
+
+ /* Process output from the file descriptors attached to the
+ children, namely log output and output path creation commands.
+ We also use this to detect child termination: if we get EOF on
+ the logger pipe of a build, we assume that the builder has
+ terminated. */
+
+ bool useTimeout = false;
+ long timeout = 0;
+ auto before = steady_time_point::clock::now();
+
+ /* If we're monitoring for silence on stdout/stderr, or if there
+ is a build timeout, then wait for input until the first
+ deadline for any child. */
+ auto nearest = steady_time_point::max(); // nearest deadline
+ if (settings.minFree.get() != 0)
+ // Periodicallty wake up to see if we need to run the garbage collector.
+ nearest = before + std::chrono::seconds(10);
+ for (auto & i : children) {
+ if (!i.respectTimeouts) continue;
+ if (0 != settings.maxSilentTime)
+ nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime));
+ if (0 != settings.buildTimeout)
+ nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout));
+ }
+ if (nearest != steady_time_point::max()) {
+ timeout = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count());
+ useTimeout = true;
+ }
+
+ /* If we are polling goals that are waiting for a lock, then wake
+ up after a few seconds at most. */
+ if (!waitingForAWhile.empty()) {
+ useTimeout = true;
+ if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before;
+ timeout = std::max(1L,
+ (long) std::chrono::duration_cast<std::chrono::seconds>(
+ lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count());
+ } else lastWokenUp = steady_time_point::min();
+
+ if (useTimeout)
+ vomit("sleeping %d seconds", timeout);
+
+ /* Use select() to wait for the input side of any logger pipe to
+ become `available'. Note that `available' (i.e., non-blocking)
+ includes EOF. */
+ std::vector<struct pollfd> pollStatus;
+ std::map <int, int> fdToPollStatus;
+ for (auto & i : children) {
+ for (auto & j : i.fds) {
+ pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
+ fdToPollStatus[j] = pollStatus.size() - 1;
+ }
+ }
+
+ if (poll(pollStatus.data(), pollStatus.size(),
+ useTimeout ? timeout * 1000 : -1) == -1) {
+ if (errno == EINTR) return;
+ throw SysError("waiting for input");
+ }
+
+ auto after = steady_time_point::clock::now();
+
+ /* Process all available file descriptors. FIXME: this is
+ O(children * fds). */
+ decltype(children)::iterator i;
+ for (auto j = children.begin(); j != children.end(); j = i) {
+ i = std::next(j);
+
+ checkInterrupt();
+
+ GoalPtr goal = j->goal.lock();
+ assert(goal);
+
+ set<int> fds2(j->fds);
+ std::vector<unsigned char> buffer(4096);
+ for (auto & k : fds2) {
+ if (pollStatus.at(fdToPollStatus.at(k)).revents) {
+ ssize_t rd = ::read(k, buffer.data(), buffer.size());
+ // FIXME: is there a cleaner way to handle pt close
+ // than EIO? Is this even standard?
+ if (rd == 0 || (rd == -1 && errno == EIO)) {
+ debug("%1%: got EOF", goal->getName());
+ goal->handleEOF(k);
+ j->fds.erase(k);
+ } else if (rd == -1) {
+ if (errno != EINTR)
+ throw SysError("%s: read failed", goal->getName());
+ } else {
+ printMsg(lvlVomit, "%1%: read %2% bytes",
+ goal->getName(), rd);
+ string data((char *) buffer.data(), rd);
+ j->lastOutput = after;
+ goal->handleChildOutput(k, data);
+ }
+ }
+ }
+
+ if (goal->exitCode == Goal::ecBusy &&
+ 0 != settings.maxSilentTime &&
+ j->respectTimeouts &&
+ after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime))
+ {
+ goal->timedOut(Error(
+ "%1% timed out after %2% seconds of silence",
+ goal->getName(), settings.maxSilentTime));
+ }
+
+ else if (goal->exitCode == Goal::ecBusy &&
+ 0 != settings.buildTimeout &&
+ j->respectTimeouts &&
+ after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout))
+ {
+ goal->timedOut(Error(
+ "%1% timed out after %2% seconds",
+ goal->getName(), settings.buildTimeout));
+ }
+ }
+
+ if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) {
+ lastWokenUp = after;
+ for (auto & i : waitingForAWhile) {
+ GoalPtr goal = i.lock();
+ if (goal) wakeUp(goal);
+ }
+ waitingForAWhile.clear();
+ }
+}
+
+
+unsigned int Worker::exitStatus()
+{
+ /*
+ * 1100100
+ * ^^^^
+ * |||`- timeout
+ * ||`-- output hash mismatch
+ * |`--- build failure
+ * `---- not deterministic
+ */
+ unsigned int mask = 0;
+ bool buildFailure = permanentFailure || timedOut || hashMismatch;
+ if (buildFailure)
+ mask |= 0x04; // 100
+ if (timedOut)
+ mask |= 0x01; // 101
+ if (hashMismatch)
+ mask |= 0x02; // 102
+ if (checkMismatch) {
+ mask |= 0x08; // 104
+ }
+
+ if (mask)
+ mask |= 0x60;
+ return mask ? mask : 1;
+}
+
+
+bool Worker::pathContentsGood(const StorePath & path)
+{
+ auto i = pathContentsGoodCache.find(path);
+ if (i != pathContentsGoodCache.end()) return i->second;
+ printInfo("checking path '%s'...", store.printStorePath(path));
+ auto info = store.queryPathInfo(path);
+ bool res;
+ if (!pathExists(store.printStorePath(path)))
+ res = false;
+ else {
+ HashResult current = hashPath(info->narHash.type, store.printStorePath(path));
+ Hash nullHash(htSHA256);
+ res = info->narHash == nullHash || info->narHash == current.first;
+ }
+ pathContentsGoodCache.insert_or_assign(path, res);
+ if (!res)
+ logError({
+ .name = "Corrupted path",
+ .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path))
+ });
+ return res;
+}
+
+
+void Worker::markContentsGood(const StorePath & path)
+{
+ pathContentsGoodCache.insert_or_assign(path, true);
+}
+
+}
diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh
new file mode 100644
index 000000000..a54316343
--- /dev/null
+++ b/src/libstore/build/worker.hh
@@ -0,0 +1,195 @@
+#pragma once
+
+#include "types.hh"
+#include "lock.hh"
+#include "local-store.hh"
+#include "goal.hh"
+
+namespace nix {
+
+/* Forward definition. */
+class DerivationGoal;
+
+typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
+
+
+/* A mapping used to remember for each child process to what goal it
+ belongs, and file descriptors for receiving log data and output
+ path creation commands. */
+struct Child
+{
+ WeakGoalPtr goal;
+ Goal * goal2; // ugly hackery
+ set<int> fds;
+ bool respectTimeouts;
+ bool inBuildSlot;
+ steady_time_point lastOutput; /* time we last got output on stdout/stderr */
+ steady_time_point timeStarted;
+};
+
+/* Forward definition. */
+struct HookInstance;
+
+/* The worker class. */
+class Worker
+{
+private:
+
+ /* Note: the worker should only have strong pointers to the
+ top-level goals. */
+
+ /* The top-level goals of the worker. */
+ Goals topGoals;
+
+ /* Goals that are ready to do some work. */
+ WeakGoals awake;
+
+ /* Goals waiting for a build slot. */
+ WeakGoals wantingToBuild;
+
+ /* Child processes currently running. */
+ std::list<Child> children;
+
+ /* Number of build slots occupied. This includes local builds and
+ substitutions but not remote builds via the build hook. */
+ unsigned int nrLocalBuilds;
+
+ /* Maps used to prevent multiple instantiations of a goal for the
+ same derivation / path. */
+ WeakGoalMap derivationGoals;
+ WeakGoalMap substitutionGoals;
+
+ /* Goals waiting for busy paths to be unlocked. */
+ WeakGoals waitingForAnyGoal;
+
+ /* Goals sleeping for a few seconds (polling a lock). */
+ WeakGoals waitingForAWhile;
+
+ /* Last time the goals in `waitingForAWhile' where woken up. */
+ steady_time_point lastWokenUp;
+
+ /* Cache for pathContentsGood(). */
+ std::map<StorePath, bool> pathContentsGoodCache;
+
+public:
+
+ const Activity act;
+ const Activity actDerivations;
+ const Activity actSubstitutions;
+
+ /* Set if at least one derivation had a BuildError (i.e. permanent
+ failure). */
+ bool permanentFailure;
+
+ /* Set if at least one derivation had a timeout. */
+ bool timedOut;
+
+ /* Set if at least one derivation fails with a hash mismatch. */
+ bool hashMismatch;
+
+ /* Set if at least one derivation is not deterministic in check mode. */
+ bool checkMismatch;
+
+ LocalStore & store;
+
+ std::unique_ptr<HookInstance> hook;
+
+ uint64_t expectedBuilds = 0;
+ uint64_t doneBuilds = 0;
+ uint64_t failedBuilds = 0;
+ uint64_t runningBuilds = 0;
+
+ uint64_t expectedSubstitutions = 0;
+ uint64_t doneSubstitutions = 0;
+ uint64_t failedSubstitutions = 0;
+ uint64_t runningSubstitutions = 0;
+ uint64_t expectedDownloadSize = 0;
+ uint64_t doneDownloadSize = 0;
+ uint64_t expectedNarSize = 0;
+ uint64_t doneNarSize = 0;
+
+ /* Whether to ask the build hook if it can build a derivation. If
+ it answers with "decline-permanently", we don't try again. */
+ bool tryBuildHook = true;
+
+ Worker(LocalStore & store);
+ ~Worker();
+
+ /* Make a goal (with caching). */
+
+ /* derivation goal */
+private:
+ std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
+ const StorePath & drvPath, const StringSet & wantedOutputs,
+ std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
+public:
+ std::shared_ptr<DerivationGoal> makeDerivationGoal(
+ const StorePath & drvPath,
+ const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
+ std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
+ const StorePath & drvPath, const BasicDerivation & drv,
+ const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
+
+ /* substitution goal */
+ GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
+
+ /* Remove a dead goal. */
+ void removeGoal(GoalPtr goal);
+
+ /* Wake up a goal (i.e., there is something for it to do). */
+ void wakeUp(GoalPtr goal);
+
+ /* Return the number of local build and substitution processes
+ currently running (but not remote builds via the build
+ hook). */
+ unsigned int getNrLocalBuilds();
+
+ /* Registers a running child process. `inBuildSlot' means that
+ the process counts towards the jobs limit. */
+ void childStarted(GoalPtr goal, const set<int> & fds,
+ bool inBuildSlot, bool respectTimeouts);
+
+ /* Unregisters a running child process. `wakeSleepers' should be
+ false if there is no sense in waking up goals that are sleeping
+ because they can't run yet (e.g., there is no free build slot,
+ or the hook would still say `postpone'). */
+ void childTerminated(Goal * goal, bool wakeSleepers = true);
+
+ /* Put `goal' to sleep until a build slot becomes available (which
+ might be right away). */
+ void waitForBuildSlot(GoalPtr goal);
+
+ /* Wait for any goal to finish. Pretty indiscriminate way to
+ wait for some resource that some other goal is holding. */
+ void waitForAnyGoal(GoalPtr goal);
+
+ /* Wait for a few seconds and then retry this goal. Used when
+ waiting for a lock held by another process. This kind of
+ polling is inefficient, but POSIX doesn't really provide a way
+ to wait for multiple locks in the main select() loop. */
+ void waitForAWhile(GoalPtr goal);
+
+ /* Loop until the specified top-level goals have finished. */
+ void run(const Goals & topGoals);
+
+ /* Wait for input to become available. */
+ void waitForInput();
+
+ unsigned int exitStatus();
+
+ /* Check whether the given valid path exists and has the right
+ contents. */
+ bool pathContentsGood(const StorePath & path);
+
+ void markContentsGood(const StorePath & path);
+
+ void updateProgress()
+ {
+ actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds);
+ actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions);
+ act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize);
+ act.setExpected(actCopyPath, expectedNarSize + doneNarSize);
+ }
+};
+
+}
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 518a357ef..bc692ca42 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -1,6 +1,7 @@
#include "derivations.hh"
#include "globals.hh"
#include "local-store.hh"
+#include "local-fs-store.hh"
#include "finally.hh"
#include <functional>
@@ -682,7 +683,7 @@ void LocalStore::removeUnusedLinks(const GCState & state)
struct stat st;
if (stat(linksDir.c_str(), &st) == -1)
throw SysError("statting '%1%'", linksDir);
- auto overhead = st.st_blocks * 512ULL;
+ int64_t overhead = st.st_blocks * 512ULL;
printInfo("note: currently hard linking saves %.2f MiB",
((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc
index 2f1d9663a..e7c3dae92 100644
--- a/src/libstore/local-fs-store.cc
+++ b/src/libstore/local-fs-store.cc
@@ -1,6 +1,7 @@
#include "archive.hh"
#include "fs-accessor.hh"
#include "store-api.hh"
+#include "local-fs-store.hh"
#include "globals.hh"
#include "compression.hh"
#include "derivations.hh"
diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh
new file mode 100644
index 000000000..8eccd8236
--- /dev/null
+++ b/src/libstore/local-fs-store.hh
@@ -0,0 +1,48 @@
+#pragma once
+
+#include "store-api.hh"
+
+namespace nix {
+
+struct LocalFSStoreConfig : virtual StoreConfig
+{
+ using StoreConfig::StoreConfig;
+ // FIXME: the (StoreConfig*) cast works around a bug in gcc that causes
+ // it to omit the call to the Setting constructor. Clang works fine
+ // either way.
+ const PathSetting rootDir{(StoreConfig*) this, true, "",
+ "root", "directory prefixed to all other paths"};
+ const PathSetting stateDir{(StoreConfig*) this, false,
+ rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir,
+ "state", "directory where Nix will store state"};
+ const PathSetting logDir{(StoreConfig*) this, false,
+ rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir,
+ "log", "directory where Nix will store state"};
+};
+
+class LocalFSStore : public virtual Store, public virtual LocalFSStoreConfig
+{
+public:
+
+ const static string drvsLogDir;
+
+ LocalFSStore(const Params & params);
+
+ void narFromPath(const StorePath & path, Sink & sink) override;
+ ref<FSAccessor> getFSAccessor() override;
+
+ /* Register a permanent GC root. */
+ Path addPermRoot(const StorePath & storePath, const Path & gcRoot);
+
+ virtual Path getRealStoreDir() { return storeDir; }
+
+ Path toRealPath(const Path & storePath) override
+ {
+ assert(isInStore(storePath));
+ return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1);
+ }
+
+ std::shared_ptr<std::string> getBuildLog(const StorePath & path) override;
+};
+
+}
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index e7c9d1605..f1e2ab7f9 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -4,6 +4,7 @@
#include "pathlocks.hh"
#include "store-api.hh"
+#include "local-fs-store.hh"
#include "sync.hh"
#include "util.hh"
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index d266c8efe..dfe1e2cc4 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -4,7 +4,7 @@ libstore_NAME = libnixstore
libstore_DIR := $(d)
-libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc)
+libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc)
libstore_LIBS = libutil
@@ -32,7 +32,7 @@ ifeq ($(HAVE_SECCOMP), 1)
endif
libstore_CXXFLAGS += \
- -I src/libutil -I src/libstore \
+ -I src/libutil -I src/libstore -I src/libstore/build \
-DNIX_PREFIX=\"$(prefix)\" \
-DNIX_STORE_DIR=\"$(storedir)\" \
-DNIX_DATA_DIR=\"$(datadir)\" \
@@ -64,3 +64,6 @@ $(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644)
$(foreach i, $(wildcard src/libstore/builtins/*.hh), \
$(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644)))
+
+$(foreach i, $(wildcard src/libstore/build/*.hh), \
+ $(eval $(call install-file-in, $(i), $(includedir)/nix/build, 0644)))
diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc
new file mode 100644
index 000000000..f1356fdca
--- /dev/null
+++ b/src/libstore/lock.cc
@@ -0,0 +1,93 @@
+#include "lock.hh"
+#include "globals.hh"
+#include "pathlocks.hh"
+
+#include <grp.h>
+#include <pwd.h>
+
+#include <fcntl.h>
+#include <unistd.h>
+
+namespace nix {
+
+UserLock::UserLock()
+{
+ assert(settings.buildUsersGroup != "");
+ createDirs(settings.nixStateDir + "/userpool");
+}
+
+bool UserLock::findFreeUser() {
+ if (enabled()) return true;
+
+ /* Get the members of the build-users-group. */
+ struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
+ if (!gr)
+ throw Error("the group '%1%' specified in 'build-users-group' does not exist",
+ settings.buildUsersGroup);
+ gid = gr->gr_gid;
+
+ /* Copy the result of getgrnam. */
+ Strings users;
+ for (char * * p = gr->gr_mem; *p; ++p) {
+ debug("found build user '%1%'", *p);
+ users.push_back(*p);
+ }
+
+ if (users.empty())
+ throw Error("the build users group '%1%' has no members",
+ settings.buildUsersGroup);
+
+ /* Find a user account that isn't currently in use for another
+ build. */
+ for (auto & i : users) {
+ debug("trying user '%1%'", i);
+
+ struct passwd * pw = getpwnam(i.c_str());
+ if (!pw)
+ throw Error("the user '%1%' in the group '%2%' does not exist",
+ i, settings.buildUsersGroup);
+
+
+ fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str();
+
+ AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
+ if (!fd)
+ throw SysError("opening user lock '%1%'", fnUserLock);
+
+ if (lockFile(fd.get(), ltWrite, false)) {
+ fdUserLock = std::move(fd);
+ user = i;
+ uid = pw->pw_uid;
+
+ /* Sanity check... */
+ if (uid == getuid() || uid == geteuid())
+ throw Error("the Nix user should not be a member of '%1%'",
+ settings.buildUsersGroup);
+
+#if __linux__
+ /* Get the list of supplementary groups of this build user. This
+ is usually either empty or contains a group such as "kvm". */
+ supplementaryGIDs.resize(10);
+ int ngroups = supplementaryGIDs.size();
+ int err = getgrouplist(pw->pw_name, pw->pw_gid,
+ supplementaryGIDs.data(), &ngroups);
+ if (err == -1)
+ throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name);
+
+ supplementaryGIDs.resize(ngroups);
+#endif
+
+ isEnabled = true;
+ return true;
+ }
+ }
+
+ return false;
+}
+
+void UserLock::kill()
+{
+ killUser(uid);
+}
+
+}
diff --git a/src/libstore/lock.hh b/src/libstore/lock.hh
new file mode 100644
index 000000000..8fbb67ddc
--- /dev/null
+++ b/src/libstore/lock.hh
@@ -0,0 +1,37 @@
+#pragma once
+
+#include "sync.hh"
+#include "types.hh"
+#include "util.hh"
+
+namespace nix {
+
+class UserLock
+{
+private:
+ Path fnUserLock;
+ AutoCloseFD fdUserLock;
+
+ bool isEnabled = false;
+ string user;
+ uid_t uid = 0;
+ gid_t gid = 0;
+ std::vector<gid_t> supplementaryGIDs;
+
+public:
+ UserLock();
+
+ void kill();
+
+ string getUser() { return user; }
+ uid_t getUID() { assert(uid); return uid; }
+ uid_t getGID() { assert(gid); return gid; }
+ std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; }
+
+ bool findFreeUser();
+
+ bool enabled() { return isEnabled; }
+
+};
+
+}
diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc
index c3809bad7..ed10dd519 100644
--- a/src/libstore/profiles.cc
+++ b/src/libstore/profiles.cc
@@ -1,5 +1,6 @@
#include "profiles.hh"
#include "store-api.hh"
+#include "local-fs-store.hh"
#include "util.hh"
#include <sys/types.h>
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 23b1942ce..488270f48 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -12,16 +12,6 @@
#include "logging.hh"
#include "callback.hh"
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/socket.h>
-#include <sys/un.h>
-#include <errno.h>
-#include <fcntl.h>
-#include <unistd.h>
-
-#include <cstring>
-
namespace nix {
namespace worker_proto {
@@ -125,69 +115,6 @@ ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper()
}
-UDSRemoteStore::UDSRemoteStore(const Params & params)
- : StoreConfig(params)
- , Store(params)
- , LocalFSStore(params)
- , RemoteStore(params)
-{
-}
-
-
-UDSRemoteStore::UDSRemoteStore(
- const std::string scheme,
- std::string socket_path,
- const Params & params)
- : UDSRemoteStore(params)
-{
- path.emplace(socket_path);
-}
-
-
-std::string UDSRemoteStore::getUri()
-{
- if (path) {
- return std::string("unix://") + *path;
- } else {
- return "daemon";
- }
-}
-
-
-ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
-{
- auto conn = make_ref<Connection>();
-
- /* Connect to a daemon that does the privileged work for us. */
- conn->fd = socket(PF_UNIX, SOCK_STREAM
- #ifdef SOCK_CLOEXEC
- | SOCK_CLOEXEC
- #endif
- , 0);
- if (!conn->fd)
- throw SysError("cannot create Unix domain socket");
- closeOnExec(conn->fd.get());
-
- string socketPath = path ? *path : settings.nixDaemonSocketFile;
-
- struct sockaddr_un addr;
- addr.sun_family = AF_UNIX;
- if (socketPath.size() + 1 >= sizeof(addr.sun_path))
- throw Error("socket path '%1%' is too long", socketPath);
- strcpy(addr.sun_path, socketPath.c_str());
-
- if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1)
- throw SysError("cannot connect to daemon at '%1%'", socketPath);
-
- conn->from.fd = conn->fd.get();
- conn->to.fd = conn->fd.get();
-
- conn->startTime = std::chrono::steady_clock::now();
-
- return conn;
-}
-
-
void RemoteStore::initConnection(Connection & conn)
{
/* Send the magic greeting, check for the reply. */
@@ -1012,6 +939,4 @@ void ConnectionHandle::withFramedSink(std::function<void(Sink &sink)> fun)
}
-static RegisterStoreImplementation<UDSRemoteStore, UDSRemoteStoreConfig> regUDSRemoteStore;
-
}
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index ec04be985..9f78fcb02 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -155,49 +155,5 @@ private:
};
-struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig
-{
- UDSRemoteStoreConfig(const Store::Params & params)
- : StoreConfig(params)
- , LocalFSStoreConfig(params)
- , RemoteStoreConfig(params)
- {
- }
-
- UDSRemoteStoreConfig()
- : UDSRemoteStoreConfig(Store::Params({}))
- {
- }
-
- const std::string name() override { return "Local Daemon Store"; }
-};
-
-class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig
-{
-public:
-
- UDSRemoteStore(const Params & params);
- UDSRemoteStore(const std::string scheme, std::string path, const Params & params);
-
- std::string getUri() override;
-
- static std::set<std::string> uriSchemes()
- { return {"unix"}; }
-
- bool sameMachine() override
- { return true; }
-
- ref<FSAccessor> getFSAccessor() override
- { return LocalFSStore::getFSAccessor(); }
-
- void narFromPath(const StorePath & path, Sink & sink) override
- { LocalFSStore::narFromPath(path, sink); }
-
-private:
-
- ref<RemoteStore::Connection> openConnection() override;
- std::optional<std::string> path;
-};
-
}
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 1bbc74db8..9f21f0434 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -1011,7 +1011,7 @@ Derivation Store::readDerivation(const StorePath & drvPath)
#include "local-store.hh"
-#include "remote-store.hh"
+#include "uds-remote-store.hh"
namespace nix {
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 450c0f554..f77bc21d1 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -715,47 +715,6 @@ protected:
};
-struct LocalFSStoreConfig : virtual StoreConfig
-{
- using StoreConfig::StoreConfig;
- // FIXME: the (StoreConfig*) cast works around a bug in gcc that causes
- // it to omit the call to the Setting constructor. Clang works fine
- // either way.
- const PathSetting rootDir{(StoreConfig*) this, true, "",
- "root", "directory prefixed to all other paths"};
- const PathSetting stateDir{(StoreConfig*) this, false,
- rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir,
- "state", "directory where Nix will store state"};
- const PathSetting logDir{(StoreConfig*) this, false,
- rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir,
- "log", "directory where Nix will store state"};
-};
-
-class LocalFSStore : public virtual Store, public virtual LocalFSStoreConfig
-{
-public:
-
- const static string drvsLogDir;
-
- LocalFSStore(const Params & params);
-
- void narFromPath(const StorePath & path, Sink & sink) override;
- ref<FSAccessor> getFSAccessor() override;
-
- /* Register a permanent GC root. */
- Path addPermRoot(const StorePath & storePath, const Path & gcRoot);
-
- virtual Path getRealStoreDir() { return storeDir; }
-
- Path toRealPath(const Path & storePath) override
- {
- assert(isInStore(storePath));
- return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1);
- }
-
- std::shared_ptr<std::string> getBuildLog(const StorePath & path) override;
-};
-
/* Copy a path from one store to another. */
void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc
new file mode 100644
index 000000000..24f3e9c6d
--- /dev/null
+++ b/src/libstore/uds-remote-store.cc
@@ -0,0 +1,81 @@
+#include "uds-remote-store.hh"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#include <cstring>
+
+
+namespace nix {
+
+UDSRemoteStore::UDSRemoteStore(const Params & params)
+ : StoreConfig(params)
+ , Store(params)
+ , LocalFSStore(params)
+ , RemoteStore(params)
+{
+}
+
+
+UDSRemoteStore::UDSRemoteStore(
+ const std::string scheme,
+ std::string socket_path,
+ const Params & params)
+ : UDSRemoteStore(params)
+{
+ path.emplace(socket_path);
+}
+
+
+std::string UDSRemoteStore::getUri()
+{
+ if (path) {
+ return std::string("unix://") + *path;
+ } else {
+ return "daemon";
+ }
+}
+
+
+ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
+{
+ auto conn = make_ref<Connection>();
+
+ /* Connect to a daemon that does the privileged work for us. */
+ conn->fd = socket(PF_UNIX, SOCK_STREAM
+ #ifdef SOCK_CLOEXEC
+ | SOCK_CLOEXEC
+ #endif
+ , 0);
+ if (!conn->fd)
+ throw SysError("cannot create Unix domain socket");
+ closeOnExec(conn->fd.get());
+
+ string socketPath = path ? *path : settings.nixDaemonSocketFile;
+
+ struct sockaddr_un addr;
+ addr.sun_family = AF_UNIX;
+ if (socketPath.size() + 1 >= sizeof(addr.sun_path))
+ throw Error("socket path '%1%' is too long", socketPath);
+ strcpy(addr.sun_path, socketPath.c_str());
+
+ if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1)
+ throw SysError("cannot connect to daemon at '%1%'", socketPath);
+
+ conn->from.fd = conn->fd.get();
+ conn->to.fd = conn->fd.get();
+
+ conn->startTime = std::chrono::steady_clock::now();
+
+ return conn;
+}
+
+
+static RegisterStoreImplementation<UDSRemoteStore, UDSRemoteStoreConfig> regUDSRemoteStore;
+
+}
diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh
new file mode 100644
index 000000000..e5de104c9
--- /dev/null
+++ b/src/libstore/uds-remote-store.hh
@@ -0,0 +1,52 @@
+#pragma once
+
+#include "remote-store.hh"
+#include "local-fs-store.hh"
+
+namespace nix {
+
+struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig
+{
+ UDSRemoteStoreConfig(const Store::Params & params)
+ : StoreConfig(params)
+ , LocalFSStoreConfig(params)
+ , RemoteStoreConfig(params)
+ {
+ }
+
+ UDSRemoteStoreConfig()
+ : UDSRemoteStoreConfig(Store::Params({}))
+ {
+ }
+
+ const std::string name() override { return "Local Daemon Store"; }
+};
+
+class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig
+{
+public:
+
+ UDSRemoteStore(const Params & params);
+ UDSRemoteStore(const std::string scheme, std::string path, const Params & params);
+
+ std::string getUri() override;
+
+ static std::set<std::string> uriSchemes()
+ { return {"unix"}; }
+
+ bool sameMachine() override
+ { return true; }
+
+ ref<FSAccessor> getFSAccessor() override
+ { return LocalFSStore::getFSAccessor(); }
+
+ void narFromPath(const StorePath & path, Sink & sink) override
+ { LocalFSStore::narFromPath(path, sink); }
+
+private:
+
+ ref<RemoteStore::Connection> openConnection() override;
+ std::optional<std::string> path;
+};
+
+}
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 2760b830b..8bd9c8aeb 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -17,8 +17,20 @@ void Args::addFlag(Flag && flag_)
if (flag->shortName) shortFlags[flag->shortName] = flag;
}
+void Completions::add(std::string completion, std::string description)
+{
+ assert(description.find('\n') == std::string::npos);
+ insert(Completion {
+ .completion = completion,
+ .description = description
+ });
+}
+
+bool Completion::operator<(const Completion & other) const
+{ return completion < other.completion || (completion == other.completion && description < other.description); }
+
bool pathCompletions = false;
-std::shared_ptr<std::set<std::string>> completions;
+std::shared_ptr<Completions> completions;
std::string completionMarker = "___COMPLETE___";
@@ -148,7 +160,7 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
for (auto & [name, flag] : longFlags) {
if (!hiddenCategories.count(flag->category)
&& hasPrefix(name, std::string(*prefix, 2)))
- completions->insert("--" + name);
+ completions->add("--" + name, flag->description);
}
}
auto i = longFlags.find(string(*pos, 2));
@@ -165,9 +177,9 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
if (auto prefix = needsCompletion(*pos)) {
if (prefix == "-") {
- completions->insert("--");
- for (auto & [flag, _] : shortFlags)
- completions->insert(std::string("-") + flag);
+ completions->add("--");
+ for (auto & [flagName, flag] : shortFlags)
+ completions->add(std::string("-") + flagName, flag->description);
}
}
@@ -244,11 +256,11 @@ nlohmann::json Args::toJSON()
return res;
}
-static void hashTypeCompleter(size_t index, std::string_view prefix)
+static void hashTypeCompleter(size_t index, std::string_view prefix)
{
for (auto & type : hashTypes)
if (hasPrefix(type, prefix))
- completions->insert(type);
+ completions->add(type);
}
Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht)
@@ -292,7 +304,7 @@ static void _completePath(std::string_view prefix, bool onlyDirs)
auto st = lstat(globbuf.gl_pathv[i]);
if (!S_ISDIR(st.st_mode)) continue;
}
- completions->insert(globbuf.gl_pathv[i]);
+ completions->add(globbuf.gl_pathv[i]);
}
globfree(&globbuf);
}
@@ -385,7 +397,7 @@ MultiCommand::MultiCommand(const Commands & commands)
if (auto prefix = needsCompletion(s)) {
for (auto & [name, command] : commands)
if (hasPrefix(name, *prefix))
- completions->insert(name);
+ completions->add(name);
}
auto i = commands.find(s);
if (i == commands.end())
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index f41242e17..26f1bc11b 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -283,7 +283,17 @@ typedef std::vector<std::pair<std::string, std::string>> Table2;
void printTable(std::ostream & out, const Table2 & table);
-extern std::shared_ptr<std::set<std::string>> completions;
+struct Completion {
+ std::string completion;
+ std::string description;
+
+ bool operator<(const Completion & other) const;
+};
+class Completions : public std::set<Completion> {
+public:
+ void add(std::string completion, std::string description = "");
+};
+extern std::shared_ptr<Completions> completions;
extern bool pathCompletions;
std::optional<std::string> needsCompletion(std::string_view s);
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 9804e9a51..53342b5cb 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1660,4 +1660,33 @@ string showBytes(uint64_t bytes)
}
+void commonChildInit(Pipe & logPipe)
+{
+ const static string pathNullDevice = "/dev/null";
+ restoreSignals();
+
+ /* Put the child in a separate session (and thus a separate
+ process group) so that it has no controlling terminal (meaning
+ that e.g. ssh cannot open /dev/tty) and it doesn't receive
+ terminal signals. */
+ if (setsid() == -1)
+ throw SysError("creating a new session");
+
+ /* Dup the write side of the logger pipe into stderr. */
+ if (dup2(logPipe.writeSide.get(), STDERR_FILENO) == -1)
+ throw SysError("cannot pipe standard error into log file");
+
+ /* Dup stderr to stdout. */
+ if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1)
+ throw SysError("cannot dup stderr into stdout");
+
+ /* Reroute stdin to /dev/null. */
+ int fdDevNull = open(pathNullDevice.c_str(), O_RDWR);
+ if (fdDevNull == -1)
+ throw SysError("cannot open '%1%'", pathNullDevice);
+ if (dup2(fdDevNull, STDIN_FILENO) == -1)
+ throw SysError("cannot dup null device into stdin");
+ close(fdDevNull);
+}
+
}
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 129d59a97..cafe93702 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -536,6 +536,8 @@ typedef std::function<bool(const Path & path)> PathFilter;
extern PathFilter defaultPathFilter;
+/* Common initialisation performed in child processes. */
+void commonChildInit(Pipe & logPipe);
/* Create a Unix domain socket in listen mode. */
AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode);
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index a79b1086b..f60e0706c 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -6,6 +6,7 @@
#include <vector>
#include "store-api.hh"
+#include "local-fs-store.hh"
#include "globals.hh"
#include "derivations.hh"
#include "affinity.hh"
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index e6667e7f5..a4b5c9e2c 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -8,6 +8,7 @@
#include "profiles.hh"
#include "shared.hh"
#include "store-api.hh"
+#include "local-fs-store.hh"
#include "user-env.hh"
#include "util.hh"
#include "json.hh"
diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc
index 8c6c8af05..87387e794 100644
--- a/src/nix-env/user-env.cc
+++ b/src/nix-env/user-env.cc
@@ -2,6 +2,7 @@
#include "util.hh"
#include "derivations.hh"
#include "store-api.hh"
+#include "local-fs-store.hh"
#include "globals.hh"
#include "shared.hh"
#include "eval.hh"
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index 18a0049a6..3956fef6d 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -8,6 +8,7 @@
#include "value-to-json.hh"
#include "util.hh"
#include "store-api.hh"
+#include "local-fs-store.hh"
#include "common-eval-args.hh"
#include "../nix/legacy.hh"
diff --git a/src/nix/build.cc b/src/nix/build.cc
index d85a482db..65708e98b 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -3,6 +3,7 @@
#include "common-args.hh"
#include "shared.hh"
#include "store-api.hh"
+#include "local-fs-store.hh"
using namespace nix;
diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc
index fc41da9e4..2d0a0b6ea 100644
--- a/src/nix/bundle.cc
+++ b/src/nix/bundle.cc
@@ -2,6 +2,7 @@
#include "common-args.hh"
#include "shared.hh"
#include "store-api.hh"
+#include "local-fs-store.hh"
#include "fs-accessor.hh"
using namespace nix;
diff --git a/src/nix/command.cc b/src/nix/command.cc
index ba7de9fdd..9a38c77f1 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -1,5 +1,6 @@
#include "command.hh"
#include "store-api.hh"
+#include "local-fs-store.hh"
#include "derivations.hh"
#include "nixexpr.hh"
#include "profiles.hh"
diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index a46ea39b6..9372f43de 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -164,6 +164,7 @@ struct Common : InstallableCommand, MixProfile
"BASHOPTS",
"EUID",
"HOME", // FIXME: don't ignore in pure mode?
+ "HOSTNAME",
"NIX_BUILD_TOP",
"NIX_ENFORCE_PURITY",
"NIX_LOG_FD",
@@ -377,6 +378,10 @@ struct CmdDevelop : Common, MixEnvironment
script += fmt("exec %s\n", concatStringsSep(" ", args));
}
+ else {
+ script += "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;\n";
+ }
+
writeFull(rcFileFd.get(), script);
stopProgressBar();
diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc
index 4588ac05e..4f3003448 100644
--- a/src/nix/doctor.cc
+++ b/src/nix/doctor.cc
@@ -5,6 +5,7 @@
#include "serve-protocol.hh"
#include "shared.hh"
#include "store-api.hh"
+#include "local-fs-store.hh"
#include "util.hh"
#include "worker-protocol.hh"
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 9bf6b7caa..7473c9758 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -26,7 +26,7 @@ void completeFlakeInputPath(
auto flake = flake::getFlake(*evalState, flakeRef, true);
for (auto & input : flake.inputs)
if (hasPrefix(input.first, prefix))
- completions->insert(input.first);
+ completions->add(input.first);
}
MixFlakeOptions::MixFlakeOptions()
@@ -211,7 +211,7 @@ void completeFlakeRefWithFragment(
auto attrPath2 = attr->getAttrPath(attr2);
/* Strip the attrpath prefix. */
attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size());
- completions->insert(flakeRefS + "#" + concatStringsSep(".", attrPath2));
+ completions->add(flakeRefS + "#" + concatStringsSep(".", attrPath2));
}
}
}
@@ -222,7 +222,7 @@ void completeFlakeRefWithFragment(
for (auto & attrPath : defaultFlakeAttrPaths) {
auto attr = root->findAlongAttrPath(parseAttrPath(*evalState, attrPath));
if (!attr) continue;
- completions->insert(flakeRefS + "#");
+ completions->add(flakeRefS + "#");
}
}
}
@@ -243,7 +243,7 @@ ref<EvalState> EvalCommand::getEvalState()
void completeFlakeRef(ref<Store> store, std::string_view prefix)
{
if (prefix == "")
- completions->insert(".");
+ completions->add(".");
completeDir(0, prefix);
@@ -254,10 +254,10 @@ void completeFlakeRef(ref<Store> store, std::string_view prefix)
if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) {
std::string from2(from, 6);
if (hasPrefix(from2, prefix))
- completions->insert(from2);
+ completions->add(from2);
} else {
if (hasPrefix(from, prefix))
- completions->insert(from);
+ completions->add(from);
}
}
}
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 1e9e07bc0..5056ceb78 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -208,7 +208,7 @@ void mainWrapped(int argc, char * * argv)
if (completions) {
std::cout << (pathCompletions ? "filenames\n" : "no-filenames\n");
for (auto & s : *completions)
- std::cout << s << "\n";
+ std::cout << s.completion << "\t" << s.description << "\n";
}
});