diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh
index 8027c61b1..3ae9a9e6b 100644
--- a/src/libstore/build/substitution-goal.hh
+++ b/src/libstore/build/substitution-goal.hh
@@ -1,681 +1,12 @@
#pragma once
-#include "machines.hh"
-#include "parsed-derivations.hh"
#include "lock.hh"
-#include "local-store.hh"
-
-#include <memory>
-#include <algorithm>
-#include <iostream>
-#include <map>
-#include <sstream>
-#include <thread>
-#include <future>
-#include <chrono>
-#include <regex>
-#include <queue>
+#include "store-api.hh"
+#include "goal.hh"
namespace nix {
-using std::map;
-
-
-/* Forward definition. */
class Worker;
-struct HookInstance;
-
-
-/* A pointer to a goal. */
-struct Goal;
-class DerivationGoal;
-typedef std::shared_ptr<Goal> GoalPtr;
-typedef std::weak_ptr<Goal> WeakGoalPtr;
-
-struct CompareGoalPtrs {
- bool operator() (const GoalPtr & a, const GoalPtr & b) const;
-};
-
-/* Set of goals. */
-typedef set<GoalPtr, CompareGoalPtrs> Goals;
-typedef list<WeakGoalPtr> WeakGoals;
-
-/* A map of paths to goals (and the other way around). */
-typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
-
-
-
-struct Goal : public std::enable_shared_from_this<Goal>
-{
- typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
-
- /* Backlink to the worker. */
- Worker & worker;
-
- /* Goals that this goal is waiting for. */
- Goals waitees;
-
- /* Goals waiting for this one to finish. Must use weak pointers
- here to prevent cycles. */
- WeakGoals waiters;
-
- /* Number of goals we are/were waiting for that have failed. */
- unsigned int nrFailed;
-
- /* Number of substitution goals we are/were waiting for that
- failed because there are no substituters. */
- unsigned int nrNoSubstituters;
-
- /* Number of substitution goals we are/were waiting for that
- failed because othey had unsubstitutable references. */
- unsigned int nrIncompleteClosure;
-
- /* Name of this goal for debugging purposes. */
- string name;
-
- /* Whether the goal is finished. */
- ExitCode exitCode;
-
- /* Exception containing an error message, if any. */
- std::optional<Error> ex;
-
- Goal(Worker & worker) : worker(worker)
- {
- nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
- exitCode = ecBusy;
- }
-
- virtual ~Goal()
- {
- trace("goal destroyed");
- }
-
- virtual void work() = 0;
-
- void addWaitee(GoalPtr waitee);
-
- virtual void waiteeDone(GoalPtr waitee, ExitCode result);
-
- virtual void handleChildOutput(int fd, const string & data)
- {
- abort();
- }
-
- virtual void handleEOF(int fd)
- {
- abort();
- }
-
- void trace(const FormatOrString & fs);
-
- string getName()
- {
- return name;
- }
-
- /* Callback in case of a timeout. It should wake up its waiters,
- get rid of any running child processes that are being monitored
- by the worker (important!), etc. */
- virtual void timedOut(Error && ex) = 0;
-
- virtual string key() = 0;
-
- void amDone(ExitCode result, std::optional<Error> ex = {});
-};
-
-typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
-
-
-/* A mapping used to remember for each child process to what goal it
- belongs, and file descriptors for receiving log data and output
- path creation commands. */
-struct Child
-{
- WeakGoalPtr goal;
- Goal * goal2; // ugly hackery
- set<int> fds;
- bool respectTimeouts;
- bool inBuildSlot;
- steady_time_point lastOutput; /* time we last got output on stdout/stderr */
- steady_time_point timeStarted;
-};
-
-
-/* The worker class. */
-class Worker
-{
-private:
-
- /* Note: the worker should only have strong pointers to the
- top-level goals. */
-
- /* The top-level goals of the worker. */
- Goals topGoals;
-
- /* Goals that are ready to do some work. */
- WeakGoals awake;
-
- /* Goals waiting for a build slot. */
- WeakGoals wantingToBuild;
-
- /* Child processes currently running. */
- std::list<Child> children;
-
- /* Number of build slots occupied. This includes local builds and
- substitutions but not remote builds via the build hook. */
- unsigned int nrLocalBuilds;
-
- /* Maps used to prevent multiple instantiations of a goal for the
- same derivation / path. */
- WeakGoalMap derivationGoals;
- WeakGoalMap substitutionGoals;
-
- /* Goals waiting for busy paths to be unlocked. */
- WeakGoals waitingForAnyGoal;
-
- /* Goals sleeping for a few seconds (polling a lock). */
- WeakGoals waitingForAWhile;
-
- /* Last time the goals in `waitingForAWhile' where woken up. */
- steady_time_point lastWokenUp;
-
- /* Cache for pathContentsGood(). */
- std::map<StorePath, bool> pathContentsGoodCache;
-
-public:
-
- const Activity act;
- const Activity actDerivations;
- const Activity actSubstitutions;
-
- /* Set if at least one derivation had a BuildError (i.e. permanent
- failure). */
- bool permanentFailure;
-
- /* Set if at least one derivation had a timeout. */
- bool timedOut;
-
- /* Set if at least one derivation fails with a hash mismatch. */
- bool hashMismatch;
-
- /* Set if at least one derivation is not deterministic in check mode. */
- bool checkMismatch;
-
- LocalStore & store;
-
- std::unique_ptr<HookInstance> hook;
-
- uint64_t expectedBuilds = 0;
- uint64_t doneBuilds = 0;
- uint64_t failedBuilds = 0;
- uint64_t runningBuilds = 0;
-
- uint64_t expectedSubstitutions = 0;
- uint64_t doneSubstitutions = 0;
- uint64_t failedSubstitutions = 0;
- uint64_t runningSubstitutions = 0;
- uint64_t expectedDownloadSize = 0;
- uint64_t doneDownloadSize = 0;
- uint64_t expectedNarSize = 0;
- uint64_t doneNarSize = 0;
-
- /* Whether to ask the build hook if it can build a derivation. If
- it answers with "decline-permanently", we don't try again. */
- bool tryBuildHook = true;
-
- Worker(LocalStore & store);
- ~Worker();
-
- /* Make a goal (with caching). */
-
- /* derivation goal */
-private:
- std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
- const StorePath & drvPath, const StringSet & wantedOutputs,
- std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
-public:
- std::shared_ptr<DerivationGoal> makeDerivationGoal(
- const StorePath & drvPath,
- const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
- std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
- const StorePath & drvPath, const BasicDerivation & drv,
- const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
-
- /* substitution goal */
- GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
-
- /* Remove a dead goal. */
- void removeGoal(GoalPtr goal);
-
- /* Wake up a goal (i.e., there is something for it to do). */
- void wakeUp(GoalPtr goal);
-
- /* Return the number of local build and substitution processes
- currently running (but not remote builds via the build
- hook). */
- unsigned int getNrLocalBuilds();
-
- /* Registers a running child process. `inBuildSlot' means that
- the process counts towards the jobs limit. */
- void childStarted(GoalPtr goal, const set<int> & fds,
- bool inBuildSlot, bool respectTimeouts);
-
- /* Unregisters a running child process. `wakeSleepers' should be
- false if there is no sense in waking up goals that are sleeping
- because they can't run yet (e.g., there is no free build slot,
- or the hook would still say `postpone'). */
- void childTerminated(Goal * goal, bool wakeSleepers = true);
-
- /* Put `goal' to sleep until a build slot becomes available (which
- might be right away). */
- void waitForBuildSlot(GoalPtr goal);
-
- /* Wait for any goal to finish. Pretty indiscriminate way to
- wait for some resource that some other goal is holding. */
- void waitForAnyGoal(GoalPtr goal);
-
- /* Wait for a few seconds and then retry this goal. Used when
- waiting for a lock held by another process. This kind of
- polling is inefficient, but POSIX doesn't really provide a way
- to wait for multiple locks in the main select() loop. */
- void waitForAWhile(GoalPtr goal);
-
- /* Loop until the specified top-level goals have finished. */
- void run(const Goals & topGoals);
-
- /* Wait for input to become available. */
- void waitForInput();
-
- unsigned int exitStatus();
-
- /* Check whether the given valid path exists and has the right
- contents. */
- bool pathContentsGood(const StorePath & path);
-
- void markContentsGood(const StorePath & path);
-
- void updateProgress()
- {
- actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds);
- actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions);
- act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize);
- act.setExpected(actCopyPath, expectedNarSize + doneNarSize);
- }
-};
-
-typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
-
-class SubstitutionGoal;
-
-/* Unless we are repairing, we don't both to test validity and just assume it,
- so the choices are `Absent` or `Valid`. */
-enum struct PathStatus {
- Corrupt,
- Absent,
- Valid,
-};
-
-struct InitialOutputStatus {
- StorePath path;
- PathStatus status;
- /* Valid in the store, and additionally non-corrupt if we are repairing */
- bool isValid() const {
- return status == PathStatus::Valid;
- }
- /* Merely present, allowed to be corrupt */
- bool isPresent() const {
- return status == PathStatus::Corrupt
- || status == PathStatus::Valid;
- }
-};
-
-struct InitialOutput {
- bool wanted;
- std::optional<InitialOutputStatus> known;
-};
-
-class DerivationGoal : public Goal
-{
-private:
- /* Whether to use an on-disk .drv file. */
- bool useDerivation;
-
- /* The path of the derivation. */
- StorePath drvPath;
-
- /* The specific outputs that we need to build. Empty means all of
- them. */
- StringSet wantedOutputs;
-
- /* Whether additional wanted outputs have been added. */
- bool needRestart = false;
-
- /* Whether to retry substituting the outputs after building the
- inputs. */
- bool retrySubstitution;
-
- /* The derivation stored at drvPath. */
- std::unique_ptr<BasicDerivation> drv;
-
- std::unique_ptr<ParsedDerivation> parsedDrv;
-
- /* The remainder is state held during the build. */
-
- /* Locks on (fixed) output paths. */
- PathLocks outputLocks;
-
- /* All input paths (that is, the union of FS closures of the
- immediate input paths). */
- StorePathSet inputPaths;
-
- std::map<std::string, InitialOutput> initialOutputs;
-
- /* User selected for running the builder. */
- std::unique_ptr<UserLock> buildUser;
-
- /* The process ID of the builder. */
- Pid pid;
-
- /* The temporary directory. */
- Path tmpDir;
-
- /* The path of the temporary directory in the sandbox. */
- Path tmpDirInSandbox;
-
- /* File descriptor for the log file. */
- AutoCloseFD fdLogFile;
- std::shared_ptr<BufferedSink> logFileSink, logSink;
-
- /* Number of bytes received from the builder's stdout/stderr. */
- unsigned long logSize;
-
- /* The most recent log lines. */
- std::list<std::string> logTail;
-
- std::string currentLogLine;
- size_t currentLogLinePos = 0; // to handle carriage return
-
- std::string currentHookLine;
-
- /* Pipe for the builder's standard output/error. */
- Pipe builderOut;
-
- /* Pipe for synchronising updates to the builder namespaces. */
- Pipe userNamespaceSync;
-
- /* The mount namespace of the builder, used to add additional
- paths to the sandbox as a result of recursive Nix calls. */
- AutoCloseFD sandboxMountNamespace;
-
- /* On Linux, whether we're doing the build in its own user
- namespace. */
- bool usingUserNamespace = true;
-
- /* The build hook. */
- std::unique_ptr<HookInstance> hook;
-
- /* Whether we're currently doing a chroot build. */
- bool useChroot = false;
-
- Path chrootRootDir;
-
- /* RAII object to delete the chroot directory. */
- std::shared_ptr<AutoDelete> autoDelChroot;
-
- /* The sort of derivation we are building. */
- DerivationType derivationType;
-
- /* Whether to run the build in a private network namespace. */
- bool privateNetwork = false;
-
- typedef void (DerivationGoal::*GoalState)();
- GoalState state;
-
- /* Stuff we need to pass to initChild(). */
- struct ChrootPath {
- Path source;
- bool optional;
- ChrootPath(Path source = "", bool optional = false)
- : source(source), optional(optional)
- { }
- };
- typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
- DirsInChroot dirsInChroot;
-
- typedef map<string, string> Environment;
- Environment env;
-
-#if __APPLE__
- typedef string SandboxProfile;
- SandboxProfile additionalSandboxProfile;
-#endif
-
- /* Hash rewriting. */
- StringMap inputRewrites, outputRewrites;
- typedef map<StorePath, StorePath> RedirectedOutputs;
- RedirectedOutputs redirectedOutputs;
-
- /* The outputs paths used during the build.
-
- - Input-addressed derivations or fixed content-addressed outputs are
- sometimes built when some of their outputs already exist, and can not
- be hidden via sandboxing. We use temporary locations instead and
- rewrite after the build. Otherwise the regular predetermined paths are
- put here.
-
- - Floating content-addressed derivations do not know their final build
- output paths until the outputs are hashed, so random locations are
- used, and then renamed. The randomness helps guard against hidden
- self-references.
- */
- OutputPathMap scratchOutputs;
-
- /* The final output paths of the build.
-
- - For input-addressed derivations, always the precomputed paths
-
- - For content-addressed derivations, calcuated from whatever the hash
- ends up being. (Note that fixed outputs derivations that produce the
- "wrong" output still install that data under its true content-address.)
- */
- OutputPathMap finalOutputs;
-
- BuildMode buildMode;
-
- /* If we're repairing without a chroot, there may be outputs that
- are valid but corrupt. So we redirect these outputs to
- temporary paths. */
- StorePathSet redirectedBadOutputs;
-
- BuildResult result;
-
- /* The current round, if we're building multiple times. */
- size_t curRound = 1;
-
- size_t nrRounds;
-
- /* Path registration info from the previous round, if we're
- building multiple times. Since this contains the hash, it
- allows us to compare whether two rounds produced the same
- result. */
- std::map<Path, ValidPathInfo> prevInfos;
-
- uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
- gid_t sandboxGid() { return usingUserNamespace ? 100 : buildUser->getGID(); }
-
- const static Path homeDir;
-
- std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
-
- std::unique_ptr<Activity> act;
-
- /* Activity that denotes waiting for a lock. */
- std::unique_ptr<Activity> actLock;
-
- std::map<ActivityId, Activity> builderActivities;
-
- /* The remote machine on which we're building. */
- std::string machineName;
-
- /* The recursive Nix daemon socket. */
- AutoCloseFD daemonSocket;
-
- /* The daemon main thread. */
- std::thread daemonThread;
-
- /* The daemon worker threads. */
- std::vector<std::thread> daemonWorkerThreads;
-
- /* Paths that were added via recursive Nix calls. */
- StorePathSet addedPaths;
-
- /* Recursive Nix calls are only allowed to build or realize paths
- in the original input closure or added via a recursive Nix call
- (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
- /nix/store/<bla> is some arbitrary path in a binary cache). */
- bool isAllowed(const StorePath & path)
- {
- return inputPaths.count(path) || addedPaths.count(path);
- }
-
- friend struct RestrictedStore;
-
-public:
- DerivationGoal(const StorePath & drvPath,
- const StringSet & wantedOutputs, Worker & worker,
- BuildMode buildMode = bmNormal);
- DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
- const StringSet & wantedOutputs, Worker & worker,
- BuildMode buildMode = bmNormal);
- ~DerivationGoal();
-
- /* Whether we need to perform hash rewriting if there are valid output paths. */
- bool needsHashRewrite();
-
- void timedOut(Error && ex) override;
-
- string key() override
- {
- /* Ensure that derivations get built in order of their name,
- i.e. a derivation named "aardvark" always comes before
- "baboon". And substitution goals always happen before
- derivation goals (due to "b$"). */
- return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
- }
-
- void work() override;
-
- StorePath getDrvPath()
- {
- return drvPath;
- }
-
- /* Add wanted outputs to an already existing derivation goal. */
- void addWantedOutputs(const StringSet & outputs);
-
- BuildResult getResult() { return result; }
-
-private:
- /* The states. */
- void getDerivation();
- void loadDerivation();
- void haveDerivation();
- void outputsSubstitutionTried();
- void gaveUpOnSubstitution();
- void closureRepaired();
- void inputsRealised();
- void tryToBuild();
- void tryLocalBuild();
- void buildDone();
-
- void resolvedFinished();
-
- /* Is the build hook willing to perform the build? */
- HookReply tryBuildHook();
-
- /* Start building a derivation. */
- void startBuilder();
-
- /* Fill in the environment for the builder. */
- void initEnv();
-
- /* Setup tmp dir location. */
- void initTmpDir();
-
- /* Write a JSON file containing the derivation attributes. */
- void writeStructuredAttrs();
-
- void startDaemon();
-
- void stopDaemon();
-
- /* Add 'path' to the set of paths that may be referenced by the
- outputs, and make it appear in the sandbox. */
- void addDependency(const StorePath & path);
-
- /* Make a file owned by the builder. */
- void chownToBuilder(const Path & path);
-
- /* Run the builder's process. */
- void runChild();
-
- friend int childEntry(void *);
-
- /* Check that the derivation outputs all exist and register them
- as valid. */
- void registerOutputs();
-
- /* Check that an output meets the requirements specified by the
- 'outputChecks' attribute (or the legacy
- '{allowed,disallowed}{References,Requisites}' attributes). */
- void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
-
- /* Open a log file and a pipe to it. */
- Path openLogFile();
-
- /* Close the log file. */
- void closeLogFile();
-
- /* Delete the temporary directory, if we have one. */
- void deleteTmpDir(bool force);
-
- /* Callback used by the worker to write to the log. */
- void handleChildOutput(int fd, const string & data) override;
- void handleEOF(int fd) override;
- void flushLine();
-
- /* Wrappers around the corresponding Store methods that first consult the
- derivation. This is currently needed because when there is no drv file
- there also is no DB entry. */
- std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
- OutputPathMap queryDerivationOutputMap();
-
- /* Return the set of (in)valid paths. */
- void checkPathValidity();
-
- /* Forcibly kill the child process, if any. */
- void killChild();
-
- /* Create alternative path calculated from but distinct from the
- input, so we can avoid overwriting outputs (or other store paths)
- that already exist. */
- StorePath makeFallbackPath(const StorePath & path);
- /* Make a path to another based on the output name along with the
- derivation hash. */
- /* FIXME add option to randomize, so we can audit whether our
- rewrites caught everything */
- StorePath makeFallbackPath(std::string_view outputName);
-
- void repairClosure();
-
- void started();
-
- void done(
- BuildResult::Status status,
- std::optional<Error> ex = {});
-
- StorePathSet exportReferences(const StorePathSet & storePaths);
-};
class SubstitutionGoal : public Goal
{
@@ -755,30 +86,4 @@ public:
StorePath getStorePath() { return storePath; }
};
-struct HookInstance
-{
- /* Pipes for talking to the build hook. */
- Pipe toHook;
-
- /* Pipe for the hook's standard output/error. */
- Pipe fromHook;
-
- /* Pipe for the builder's standard output/error. */
- Pipe builderOut;
-
- /* The process ID of the hook. */
- Pid pid;
-
- FdSink sink;
-
- std::map<ActivityId, Activity> activities;
-
- HookInstance();
-
- ~HookInstance();
-};
-
-
-void addToWeakGoals(WeakGoals & goals, GoalPtr p);
-
}