Trim derivation-goal.hh
This commit is contained in:
John Ericson
parent
4eb8c69853
commit
2ce726947a
1 changed files with 2 additions and 407 deletions
|
|
@ -1,312 +1,18 @@
|
|||
#pragma once
|
||||
|
||||
#include "machines.hh"
|
||||
#include "parsed-derivations.hh"
|
||||
#include "lock.hh"
|
||||
#include "local-store.hh"
|
||||
|
||||
#include <memory>
|
||||
#include <algorithm>
|
||||
#include <iostream>
|
||||
#include <map>
|
||||
#include <sstream>
|
||||
#include <thread>
|
||||
#include <future>
|
||||
#include <chrono>
|
||||
#include <regex>
|
||||
#include <queue>
|
||||
#include "goal.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
using std::map;
|
||||
|
||||
|
||||
/* Forward definition. */
|
||||
class Worker;
|
||||
struct HookInstance;
|
||||
|
||||
|
||||
/* A pointer to a goal. */
|
||||
struct Goal;
|
||||
class DerivationGoal;
|
||||
typedef std::shared_ptr<Goal> GoalPtr;
|
||||
typedef std::weak_ptr<Goal> WeakGoalPtr;
|
||||
|
||||
struct CompareGoalPtrs {
|
||||
bool operator() (const GoalPtr & a, const GoalPtr & b) const;
|
||||
};
|
||||
|
||||
/* Set of goals. */
|
||||
typedef set<GoalPtr, CompareGoalPtrs> Goals;
|
||||
typedef list<WeakGoalPtr> WeakGoals;
|
||||
|
||||
/* A map of paths to goals (and the other way around). */
|
||||
typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
|
||||
|
||||
|
||||
|
||||
struct Goal : public std::enable_shared_from_this<Goal>
|
||||
{
|
||||
typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
|
||||
|
||||
/* Backlink to the worker. */
|
||||
Worker & worker;
|
||||
|
||||
/* Goals that this goal is waiting for. */
|
||||
Goals waitees;
|
||||
|
||||
/* Goals waiting for this one to finish. Must use weak pointers
|
||||
here to prevent cycles. */
|
||||
WeakGoals waiters;
|
||||
|
||||
/* Number of goals we are/were waiting for that have failed. */
|
||||
unsigned int nrFailed;
|
||||
|
||||
/* Number of substitution goals we are/were waiting for that
|
||||
failed because there are no substituters. */
|
||||
unsigned int nrNoSubstituters;
|
||||
|
||||
/* Number of substitution goals we are/were waiting for that
|
||||
failed because othey had unsubstitutable references. */
|
||||
unsigned int nrIncompleteClosure;
|
||||
|
||||
/* Name of this goal for debugging purposes. */
|
||||
string name;
|
||||
|
||||
/* Whether the goal is finished. */
|
||||
ExitCode exitCode;
|
||||
|
||||
/* Exception containing an error message, if any. */
|
||||
std::optional<Error> ex;
|
||||
|
||||
Goal(Worker & worker) : worker(worker)
|
||||
{
|
||||
nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
|
||||
exitCode = ecBusy;
|
||||
}
|
||||
|
||||
virtual ~Goal()
|
||||
{
|
||||
trace("goal destroyed");
|
||||
}
|
||||
|
||||
virtual void work() = 0;
|
||||
|
||||
void addWaitee(GoalPtr waitee);
|
||||
|
||||
virtual void waiteeDone(GoalPtr waitee, ExitCode result);
|
||||
|
||||
virtual void handleChildOutput(int fd, const string & data)
|
||||
{
|
||||
abort();
|
||||
}
|
||||
|
||||
virtual void handleEOF(int fd)
|
||||
{
|
||||
abort();
|
||||
}
|
||||
|
||||
void trace(const FormatOrString & fs);
|
||||
|
||||
string getName()
|
||||
{
|
||||
return name;
|
||||
}
|
||||
|
||||
/* Callback in case of a timeout. It should wake up its waiters,
|
||||
get rid of any running child processes that are being monitored
|
||||
by the worker (important!), etc. */
|
||||
virtual void timedOut(Error && ex) = 0;
|
||||
|
||||
virtual string key() = 0;
|
||||
|
||||
void amDone(ExitCode result, std::optional<Error> ex = {});
|
||||
};
|
||||
|
||||
typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
|
||||
|
||||
|
||||
/* A mapping used to remember for each child process to what goal it
|
||||
belongs, and file descriptors for receiving log data and output
|
||||
path creation commands. */
|
||||
struct Child
|
||||
{
|
||||
WeakGoalPtr goal;
|
||||
Goal * goal2; // ugly hackery
|
||||
set<int> fds;
|
||||
bool respectTimeouts;
|
||||
bool inBuildSlot;
|
||||
steady_time_point lastOutput; /* time we last got output on stdout/stderr */
|
||||
steady_time_point timeStarted;
|
||||
};
|
||||
|
||||
|
||||
/* The worker class. */
|
||||
class Worker
|
||||
{
|
||||
private:
|
||||
|
||||
/* Note: the worker should only have strong pointers to the
|
||||
top-level goals. */
|
||||
|
||||
/* The top-level goals of the worker. */
|
||||
Goals topGoals;
|
||||
|
||||
/* Goals that are ready to do some work. */
|
||||
WeakGoals awake;
|
||||
|
||||
/* Goals waiting for a build slot. */
|
||||
WeakGoals wantingToBuild;
|
||||
|
||||
/* Child processes currently running. */
|
||||
std::list<Child> children;
|
||||
|
||||
/* Number of build slots occupied. This includes local builds and
|
||||
substitutions but not remote builds via the build hook. */
|
||||
unsigned int nrLocalBuilds;
|
||||
|
||||
/* Maps used to prevent multiple instantiations of a goal for the
|
||||
same derivation / path. */
|
||||
WeakGoalMap derivationGoals;
|
||||
WeakGoalMap substitutionGoals;
|
||||
|
||||
/* Goals waiting for busy paths to be unlocked. */
|
||||
WeakGoals waitingForAnyGoal;
|
||||
|
||||
/* Goals sleeping for a few seconds (polling a lock). */
|
||||
WeakGoals waitingForAWhile;
|
||||
|
||||
/* Last time the goals in `waitingForAWhile' where woken up. */
|
||||
steady_time_point lastWokenUp;
|
||||
|
||||
/* Cache for pathContentsGood(). */
|
||||
std::map<StorePath, bool> pathContentsGoodCache;
|
||||
|
||||
public:
|
||||
|
||||
const Activity act;
|
||||
const Activity actDerivations;
|
||||
const Activity actSubstitutions;
|
||||
|
||||
/* Set if at least one derivation had a BuildError (i.e. permanent
|
||||
failure). */
|
||||
bool permanentFailure;
|
||||
|
||||
/* Set if at least one derivation had a timeout. */
|
||||
bool timedOut;
|
||||
|
||||
/* Set if at least one derivation fails with a hash mismatch. */
|
||||
bool hashMismatch;
|
||||
|
||||
/* Set if at least one derivation is not deterministic in check mode. */
|
||||
bool checkMismatch;
|
||||
|
||||
LocalStore & store;
|
||||
|
||||
std::unique_ptr<HookInstance> hook;
|
||||
|
||||
uint64_t expectedBuilds = 0;
|
||||
uint64_t doneBuilds = 0;
|
||||
uint64_t failedBuilds = 0;
|
||||
uint64_t runningBuilds = 0;
|
||||
|
||||
uint64_t expectedSubstitutions = 0;
|
||||
uint64_t doneSubstitutions = 0;
|
||||
uint64_t failedSubstitutions = 0;
|
||||
uint64_t runningSubstitutions = 0;
|
||||
uint64_t expectedDownloadSize = 0;
|
||||
uint64_t doneDownloadSize = 0;
|
||||
uint64_t expectedNarSize = 0;
|
||||
uint64_t doneNarSize = 0;
|
||||
|
||||
/* Whether to ask the build hook if it can build a derivation. If
|
||||
it answers with "decline-permanently", we don't try again. */
|
||||
bool tryBuildHook = true;
|
||||
|
||||
Worker(LocalStore & store);
|
||||
~Worker();
|
||||
|
||||
/* Make a goal (with caching). */
|
||||
|
||||
/* derivation goal */
|
||||
private:
|
||||
std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
|
||||
const StorePath & drvPath, const StringSet & wantedOutputs,
|
||||
std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
|
||||
public:
|
||||
std::shared_ptr<DerivationGoal> makeDerivationGoal(
|
||||
const StorePath & drvPath,
|
||||
const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
|
||||
std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
|
||||
const StorePath & drvPath, const BasicDerivation & drv,
|
||||
const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
|
||||
|
||||
/* substitution goal */
|
||||
GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
|
||||
|
||||
/* Remove a dead goal. */
|
||||
void removeGoal(GoalPtr goal);
|
||||
|
||||
/* Wake up a goal (i.e., there is something for it to do). */
|
||||
void wakeUp(GoalPtr goal);
|
||||
|
||||
/* Return the number of local build and substitution processes
|
||||
currently running (but not remote builds via the build
|
||||
hook). */
|
||||
unsigned int getNrLocalBuilds();
|
||||
|
||||
/* Registers a running child process. `inBuildSlot' means that
|
||||
the process counts towards the jobs limit. */
|
||||
void childStarted(GoalPtr goal, const set<int> & fds,
|
||||
bool inBuildSlot, bool respectTimeouts);
|
||||
|
||||
/* Unregisters a running child process. `wakeSleepers' should be
|
||||
false if there is no sense in waking up goals that are sleeping
|
||||
because they can't run yet (e.g., there is no free build slot,
|
||||
or the hook would still say `postpone'). */
|
||||
void childTerminated(Goal * goal, bool wakeSleepers = true);
|
||||
|
||||
/* Put `goal' to sleep until a build slot becomes available (which
|
||||
might be right away). */
|
||||
void waitForBuildSlot(GoalPtr goal);
|
||||
|
||||
/* Wait for any goal to finish. Pretty indiscriminate way to
|
||||
wait for some resource that some other goal is holding. */
|
||||
void waitForAnyGoal(GoalPtr goal);
|
||||
|
||||
/* Wait for a few seconds and then retry this goal. Used when
|
||||
waiting for a lock held by another process. This kind of
|
||||
polling is inefficient, but POSIX doesn't really provide a way
|
||||
to wait for multiple locks in the main select() loop. */
|
||||
void waitForAWhile(GoalPtr goal);
|
||||
|
||||
/* Loop until the specified top-level goals have finished. */
|
||||
void run(const Goals & topGoals);
|
||||
|
||||
/* Wait for input to become available. */
|
||||
void waitForInput();
|
||||
|
||||
unsigned int exitStatus();
|
||||
|
||||
/* Check whether the given valid path exists and has the right
|
||||
contents. */
|
||||
bool pathContentsGood(const StorePath & path);
|
||||
|
||||
void markContentsGood(const StorePath & path);
|
||||
|
||||
void updateProgress()
|
||||
{
|
||||
actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds);
|
||||
actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions);
|
||||
act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize);
|
||||
act.setExpected(actCopyPath, expectedNarSize + doneNarSize);
|
||||
}
|
||||
};
|
||||
|
||||
typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
|
||||
|
||||
class SubstitutionGoal;
|
||||
|
||||
/* Unless we are repairing, we don't both to test validity and just assume it,
|
||||
so the choices are `Absent` or `Valid`. */
|
||||
enum struct PathStatus {
|
||||
|
|
@ -554,14 +260,7 @@ public:
|
|||
|
||||
void timedOut(Error && ex) override;
|
||||
|
||||
string key() override
|
||||
{
|
||||
/* Ensure that derivations get built in order of their name,
|
||||
i.e. a derivation named "aardvark" always comes before
|
||||
"baboon". And substitution goals always happen before
|
||||
derivation goals (due to "b$"). */
|
||||
return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
|
||||
}
|
||||
string key() override;
|
||||
|
||||
void work() override;
|
||||
|
||||
|
|
@ -677,108 +376,4 @@ private:
|
|||
StorePathSet exportReferences(const StorePathSet & storePaths);
|
||||
};
|
||||
|
||||
class SubstitutionGoal : public Goal
|
||||
{
|
||||
friend class Worker;
|
||||
|
||||
private:
|
||||
/* The store path that should be realised through a substitute. */
|
||||
StorePath storePath;
|
||||
|
||||
/* The path the substituter refers to the path as. This will be
|
||||
* different when the stores have different names. */
|
||||
std::optional<StorePath> subPath;
|
||||
|
||||
/* The remaining substituters. */
|
||||
std::list<ref<Store>> subs;
|
||||
|
||||
/* The current substituter. */
|
||||
std::shared_ptr<Store> sub;
|
||||
|
||||
/* Whether a substituter failed. */
|
||||
bool substituterFailed = false;
|
||||
|
||||
/* Path info returned by the substituter's query info operation. */
|
||||
std::shared_ptr<const ValidPathInfo> info;
|
||||
|
||||
/* Pipe for the substituter's standard output. */
|
||||
Pipe outPipe;
|
||||
|
||||
/* The substituter thread. */
|
||||
std::thread thr;
|
||||
|
||||
std::promise<void> promise;
|
||||
|
||||
/* Whether to try to repair a valid path. */
|
||||
RepairFlag repair;
|
||||
|
||||
/* Location where we're downloading the substitute. Differs from
|
||||
storePath when doing a repair. */
|
||||
Path destPath;
|
||||
|
||||
std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
|
||||
maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
|
||||
|
||||
typedef void (SubstitutionGoal::*GoalState)();
|
||||
GoalState state;
|
||||
|
||||
/* Content address for recomputing store path */
|
||||
std::optional<ContentAddress> ca;
|
||||
|
||||
public:
|
||||
SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
|
||||
~SubstitutionGoal();
|
||||
|
||||
void timedOut(Error && ex) override { abort(); };
|
||||
|
||||
string key() override
|
||||
{
|
||||
/* "a$" ensures substitution goals happen before derivation
|
||||
goals. */
|
||||
return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
|
||||
}
|
||||
|
||||
void work() override;
|
||||
|
||||
/* The states. */
|
||||
void init();
|
||||
void tryNext();
|
||||
void gotInfo();
|
||||
void referencesValid();
|
||||
void tryToRun();
|
||||
void finished();
|
||||
|
||||
/* Callback used by the worker to write to the log. */
|
||||
void handleChildOutput(int fd, const string & data) override;
|
||||
void handleEOF(int fd) override;
|
||||
|
||||
StorePath getStorePath() { return storePath; }
|
||||
};
|
||||
|
||||
struct HookInstance
|
||||
{
|
||||
/* Pipes for talking to the build hook. */
|
||||
Pipe toHook;
|
||||
|
||||
/* Pipe for the hook's standard output/error. */
|
||||
Pipe fromHook;
|
||||
|
||||
/* Pipe for the builder's standard output/error. */
|
||||
Pipe builderOut;
|
||||
|
||||
/* The process ID of the hook. */
|
||||
Pid pid;
|
||||
|
||||
FdSink sink;
|
||||
|
||||
std::map<ActivityId, Activity> activities;
|
||||
|
||||
HookInstance();
|
||||
|
||||
~HookInstance();
|
||||
};
|
||||
|
||||
|
||||
void addToWeakGoals(WeakGoals & goals, GoalPtr p);
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue