* Make nix-env --dry-run print the paths to be substituted correctly
again. (After the previous substituter mechanism refactoring I didn't update the code that obtains the references of substitutable paths.) This required some refactoring: the substituter programs are now kept running and receive/respond to info requests via stdin/stdout.
This commit is contained in:
Eelco Dolstra
parent
fc691e1cbd
commit
3c92ea399d
14 changed files with 338 additions and 272 deletions
|
|
@ -2,6 +2,9 @@
|
|||
|
||||
use strict;
|
||||
use File::Basename;
|
||||
use IO::Handle;
|
||||
|
||||
STDOUT->autoflush(1);
|
||||
|
||||
my @remoteStoresAll = split ':', ($ENV{"NIX_OTHER_STORES"} or "");
|
||||
|
||||
|
|
@ -33,42 +36,46 @@ sub findStorePath {
|
|||
}
|
||||
|
||||
|
||||
if ($ARGV[0] eq "--query-paths") {
|
||||
foreach my $store (@remoteStores) {
|
||||
opendir DIR, "$store/var/nix/db/info" or next;
|
||||
print "@storedir@/$_\n" foreach readdir DIR;
|
||||
closedir DIR;
|
||||
}
|
||||
}
|
||||
if ($ARGV[0] eq "--query") {
|
||||
|
||||
while (<STDIN>) {
|
||||
my $cmd = $_; chomp $cmd;
|
||||
|
||||
elsif ($ARGV[0] eq "--query-info") {
|
||||
shift @ARGV;
|
||||
|
||||
foreach my $storePath (@ARGV) {
|
||||
|
||||
(my $infoFile) = findStorePath $storePath;
|
||||
next unless $infoFile;
|
||||
|
||||
my $deriver = "";
|
||||
my @references = ();
|
||||
|
||||
open INFO, "<$infoFile" or die "cannot read info file $infoFile\n";
|
||||
while (<INFO>) {
|
||||
chomp;
|
||||
#print STDERR "GOT $_\n";
|
||||
/^([\w-]+): (.*)$/ or die "bad info file";
|
||||
my $key = $1;
|
||||
my $value = $2;
|
||||
if ($key eq "Deriver") { $deriver = $value; }
|
||||
elsif ($key eq "References") { @references = split ' ', $value; }
|
||||
if ($cmd eq "have") {
|
||||
my $storePath = <STDIN>; chomp $storePath;
|
||||
(my $infoFile) = findStorePath $storePath;
|
||||
print STDOUT ($infoFile ? "1\n" : "0\n");
|
||||
}
|
||||
close INFO;
|
||||
|
||||
print "$storePath\n";
|
||||
print "$deriver\n";
|
||||
print scalar @references, "\n";
|
||||
print "$_\n" foreach @references;
|
||||
elsif ($cmd eq "info") {
|
||||
my $storePath = <STDIN>; chomp $storePath;
|
||||
(my $infoFile) = findStorePath $storePath;
|
||||
if (!$infoFile) {
|
||||
print "0\n";
|
||||
next; # not an error
|
||||
}
|
||||
print "1\n";
|
||||
|
||||
my $deriver = "";
|
||||
my @references = ();
|
||||
|
||||
open INFO, "<$infoFile" or die "cannot read info file $infoFile\n";
|
||||
while (<INFO>) {
|
||||
chomp;
|
||||
/^([\w-]+): (.*)$/ or die "bad info file";
|
||||
my $key = $1;
|
||||
my $value = $2;
|
||||
if ($key eq "Deriver") { $deriver = $value; }
|
||||
elsif ($key eq "References") { @references = split ' ', $value; }
|
||||
}
|
||||
close INFO;
|
||||
|
||||
print "$deriver\n";
|
||||
print scalar @references, "\n";
|
||||
print "$_\n" foreach @references;
|
||||
}
|
||||
|
||||
else { die "unknown command `$cmd'"; }
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -5,27 +5,18 @@ use readmanifest;
|
|||
use POSIX qw(strftime);
|
||||
use File::Temp qw(tempdir);
|
||||
|
||||
STDOUT->autoflush(1);
|
||||
|
||||
my $manifestDir = "@localstatedir@/nix/manifests";
|
||||
my $logFile = "@localstatedir@/log/nix/downloads";
|
||||
|
||||
|
||||
# Create a temporary directory.
|
||||
my $tmpDir = tempdir("nix-download.XXXXXX", CLEANUP => 1, TMPDIR => 1)
|
||||
or die "cannot create a temporary directory";
|
||||
|
||||
chdir $tmpDir or die "cannot change to `$tmpDir': $!";
|
||||
|
||||
my $tmpNar = "$tmpDir/nar";
|
||||
my $tmpNar2 = "$tmpDir/nar2";
|
||||
|
||||
|
||||
# Load all manifests.
|
||||
my %narFiles;
|
||||
my %localPaths;
|
||||
my %patches;
|
||||
|
||||
for my $manifest (glob "$manifestDir/*.nixmanifest") {
|
||||
# print STDERR "reading $manifest\n";
|
||||
if (readManifest($manifest, \%narFiles, \%localPaths, \%patches) < 3) {
|
||||
print STDERR "you have an old-style manifest `$manifest'; please delete it\n";
|
||||
exit 1;
|
||||
|
|
@ -35,34 +26,40 @@ for my $manifest (glob "$manifestDir/*.nixmanifest") {
|
|||
|
||||
# Parse the arguments.
|
||||
|
||||
if ($ARGV[0] eq "--query-paths") {
|
||||
foreach my $storePath (keys %narFiles) { print "$storePath\n"; }
|
||||
foreach my $storePath (keys %localPaths) { print "$storePath\n"; }
|
||||
exit 0;
|
||||
}
|
||||
if ($ARGV[0] eq "--query") {
|
||||
|
||||
elsif ($ARGV[0] eq "--query-info") {
|
||||
shift @ARGV;
|
||||
foreach my $storePath (@ARGV) {
|
||||
my $info;
|
||||
if (defined $narFiles{$storePath}) {
|
||||
$info = @{$narFiles{$storePath}}[0];
|
||||
while (<STDIN>) {
|
||||
my $cmd = $_; chomp $cmd;
|
||||
|
||||
if ($cmd eq "have") {
|
||||
my $storePath = <STDIN>; chomp $storePath;
|
||||
print STDOUT ((defined $narFiles{$storePath} or defined $localPaths{$storePath})
|
||||
? "1\n" : "0\n");
|
||||
}
|
||||
elsif (defined $localPaths{$storePath}) {
|
||||
$info = @{$localPaths{$storePath}}[0];
|
||||
}
|
||||
else {
|
||||
next; # not an error
|
||||
}
|
||||
print "$storePath\n";
|
||||
print "$info->{deriver}\n";
|
||||
my @references = split " ", $info->{references};
|
||||
my $count = scalar @references;
|
||||
print "$count\n";
|
||||
foreach my $reference (@references) {
|
||||
print "$reference\n";
|
||||
|
||||
elsif ($cmd eq "info") {
|
||||
my $storePath = <STDIN>; chomp $storePath;
|
||||
my $info;
|
||||
if (defined $narFiles{$storePath}) {
|
||||
$info = @{$narFiles{$storePath}}[0];
|
||||
}
|
||||
elsif (defined $localPaths{$storePath}) {
|
||||
$info = @{$localPaths{$storePath}}[0];
|
||||
}
|
||||
else {
|
||||
print "0\n";
|
||||
next; # not an error
|
||||
}
|
||||
print "1\n";
|
||||
print "$info->{deriver}\n";
|
||||
my @references = split " ", $info->{references};
|
||||
print scalar @references, "\n";
|
||||
print "$_\n" foreach @references;
|
||||
}
|
||||
|
||||
else { die "unknown command `$cmd'"; }
|
||||
}
|
||||
|
||||
exit 0;
|
||||
}
|
||||
|
||||
|
|
@ -75,6 +72,16 @@ die unless scalar @ARGV == 2;
|
|||
my $targetPath = $ARGV[1];
|
||||
|
||||
|
||||
# Create a temporary directory.
|
||||
my $tmpDir = tempdir("nix-download.XXXXXX", CLEANUP => 1, TMPDIR => 1)
|
||||
or die "cannot create a temporary directory";
|
||||
|
||||
chdir $tmpDir or die "cannot change to `$tmpDir': $!";
|
||||
|
||||
my $tmpNar = "$tmpDir/nar";
|
||||
my $tmpNar2 = "$tmpDir/nar2";
|
||||
|
||||
|
||||
open LOGFILE, ">>$logFile" or die "cannot open log file $logFile";
|
||||
|
||||
my $date = strftime ("%F %H:%M:%S UTC", gmtime (time));
|
||||
|
|
|
|||
|
|
@ -184,11 +184,6 @@ private:
|
|||
/* Goals waiting for a build slot. */
|
||||
WeakGoals wantingToBuild;
|
||||
|
||||
/* Goals waiting for info from substituters (using --query-info),
|
||||
and the info they're (collectively) waiting for. */
|
||||
WeakGoals waitingForInfo;
|
||||
map<Path, PathSet> requestedInfo;
|
||||
|
||||
/* Child processes currently running. */
|
||||
Children children;
|
||||
|
||||
|
|
@ -243,11 +238,6 @@ public:
|
|||
call is made to childTerminate(..., true). */
|
||||
void waitForChildTermination(GoalPtr goal);
|
||||
|
||||
/* Put `goal' to sleep until the top-level loop has run `sub' to
|
||||
get info about `storePath' (with --query-info). We combine
|
||||
substituter invocations to reduce overhead. */
|
||||
void waitForInfo(GoalPtr goal, Path sub, Path storePath);
|
||||
|
||||
/* Wait for any goal to finish. Pretty indiscriminate way to
|
||||
wait for some resource that some other goal is holding. */
|
||||
void waitForAnyGoal(GoalPtr goal);
|
||||
|
|
@ -257,12 +247,6 @@ public:
|
|||
|
||||
/* Wait for input to become available. */
|
||||
void waitForInput();
|
||||
|
||||
private:
|
||||
|
||||
/* Process the pending paths in requestedInfo and wake up the
|
||||
goals in waitingForInfo. */
|
||||
void getInfo();
|
||||
|
||||
};
|
||||
|
||||
|
|
@ -2042,12 +2026,12 @@ void DerivationGoal::initChild()
|
|||
}
|
||||
|
||||
/* Close all other file descriptors. */
|
||||
int maxFD = 0;
|
||||
maxFD = sysconf(_SC_OPEN_MAX);
|
||||
for (int fd = 0; fd < maxFD; ++fd)
|
||||
if (fd != STDIN_FILENO && fd != STDOUT_FILENO && fd != STDERR_FILENO
|
||||
&& (!inHook || (fd != 3 && fd != 4)))
|
||||
close(fd); /* ignore result */
|
||||
set<int> exceptions;
|
||||
if (inHook) {
|
||||
exceptions.insert(3);
|
||||
exceptions.insert(4);
|
||||
}
|
||||
closeMostFDs(exceptions);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -2118,10 +2102,8 @@ private:
|
|||
/* The current substituter. */
|
||||
Path sub;
|
||||
|
||||
/* Path info returned by the substituter's --query-info operation. */
|
||||
bool infoOkay;
|
||||
PathSet references;
|
||||
Path deriver;
|
||||
/* Path info returned by the substituter's query info operation. */
|
||||
SubstitutablePathInfo info;
|
||||
|
||||
/* Pipe for the substitute's standard output/error. */
|
||||
Pipe logPipe;
|
||||
|
|
@ -2205,6 +2187,42 @@ void SubstitutionGoal::init()
|
|||
return;
|
||||
}
|
||||
|
||||
if (!worker.store.querySubstitutablePathInfo(storePath, info)) {
|
||||
printMsg(lvlError,
|
||||
format("path `%1%' is required, but there is no substituter that knows anything about it")
|
||||
% storePath);
|
||||
amDone(ecFailed);
|
||||
return;
|
||||
}
|
||||
|
||||
/* To maintain the closure invariant, we first have to realise the
|
||||
paths referenced by this one. */
|
||||
foreach (PathSet::iterator, i, info.references)
|
||||
if (*i != storePath) /* ignore self-references */
|
||||
addWaitee(worker.makeSubstitutionGoal(*i));
|
||||
|
||||
if (waitees.empty()) /* to prevent hang (no wake-up event) */
|
||||
referencesValid();
|
||||
else
|
||||
state = &SubstitutionGoal::referencesValid;
|
||||
}
|
||||
|
||||
|
||||
void SubstitutionGoal::referencesValid()
|
||||
{
|
||||
trace("all references realised");
|
||||
|
||||
if (nrFailed > 0) {
|
||||
printMsg(lvlError,
|
||||
format("some references of path `%1%' could not be realised") % storePath);
|
||||
amDone(ecFailed);
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (PathSet::iterator, i, info.references)
|
||||
if (*i != storePath) /* ignore self-references */
|
||||
assert(worker.store.isValidPath(*i));
|
||||
|
||||
subs = substituters;
|
||||
|
||||
tryNext();
|
||||
|
|
@ -2228,51 +2246,6 @@ void SubstitutionGoal::tryNext()
|
|||
sub = subs.front();
|
||||
subs.pop_front();
|
||||
|
||||
infoOkay = false;
|
||||
state = &SubstitutionGoal::gotInfo;
|
||||
worker.waitForInfo(shared_from_this(), sub, storePath);
|
||||
}
|
||||
|
||||
|
||||
void SubstitutionGoal::gotInfo()
|
||||
{
|
||||
trace("got info");
|
||||
|
||||
if (!infoOkay) {
|
||||
tryNext();
|
||||
return;
|
||||
}
|
||||
|
||||
/* To maintain the closure invariant, we first have to realise the
|
||||
paths referenced by this one. */
|
||||
for (PathSet::iterator i = references.begin();
|
||||
i != references.end(); ++i)
|
||||
if (*i != storePath) /* ignore self-references */
|
||||
addWaitee(worker.makeSubstitutionGoal(*i));
|
||||
|
||||
if (waitees.empty()) /* to prevent hang (no wake-up event) */
|
||||
referencesValid();
|
||||
else
|
||||
state = &SubstitutionGoal::referencesValid;
|
||||
}
|
||||
|
||||
|
||||
void SubstitutionGoal::referencesValid()
|
||||
{
|
||||
trace("all references realised");
|
||||
|
||||
if (nrFailed > 0) {
|
||||
printMsg(lvlError,
|
||||
format("some references of path `%1%' could not be realised") % storePath);
|
||||
amDone(ecFailed);
|
||||
return;
|
||||
}
|
||||
|
||||
for (PathSet::iterator i = references.begin();
|
||||
i != references.end(); ++i)
|
||||
if (*i != storePath) /* ignore self-references */
|
||||
assert(worker.store.isValidPath(*i));
|
||||
|
||||
state = &SubstitutionGoal::tryToRun;
|
||||
worker.waitForBuildSlot(shared_from_this());
|
||||
}
|
||||
|
|
@ -2413,7 +2386,7 @@ void SubstitutionGoal::finished()
|
|||
Hash contentHash = hashPath(htSHA256, storePath);
|
||||
|
||||
worker.store.registerValidPath(storePath, contentHash,
|
||||
references, deriver);
|
||||
info.references, info.deriver);
|
||||
|
||||
outputLock->setDeletion(true);
|
||||
|
||||
|
|
@ -2608,14 +2581,6 @@ void Worker::waitForChildTermination(GoalPtr goal)
|
|||
}
|
||||
|
||||
|
||||
void Worker::waitForInfo(GoalPtr goal, Path sub, Path storePath)
|
||||
{
|
||||
debug("wait for info");
|
||||
requestedInfo[sub].insert(storePath);
|
||||
waitingForInfo.insert(goal);
|
||||
}
|
||||
|
||||
|
||||
void Worker::waitForAnyGoal(GoalPtr goal)
|
||||
{
|
||||
debug("wait for any goal");
|
||||
|
|
@ -2623,68 +2588,6 @@ void Worker::waitForAnyGoal(GoalPtr goal)
|
|||
}
|
||||
|
||||
|
||||
void Worker::getInfo()
|
||||
{
|
||||
for (map<Path, PathSet>::iterator i = requestedInfo.begin();
|
||||
i != requestedInfo.end(); ++i)
|
||||
{
|
||||
Path sub = i->first;
|
||||
PathSet paths = i->second;
|
||||
|
||||
while (!paths.empty()) {
|
||||
|
||||
/* Run the substituter for at most 100 paths at a time to
|
||||
prevent command line overflows. */
|
||||
PathSet paths2;
|
||||
while (!paths.empty() && paths2.size() < 100) {
|
||||
paths2.insert(*paths.begin());
|
||||
paths.erase(paths.begin());
|
||||
}
|
||||
|
||||
/* Ask the substituter for the references and deriver of
|
||||
the paths. */
|
||||
debug(format("running `%1%' to get info about `%2%'") % sub % showPaths(paths2));
|
||||
Strings args;
|
||||
args.push_back("--query-info");
|
||||
args.insert(args.end(), paths2.begin(), paths2.end());
|
||||
string res = runProgram(sub, false, args);
|
||||
std::istringstream str(res);
|
||||
|
||||
while (true) {
|
||||
ValidPathInfo info = decodeValidPathInfo(str);
|
||||
if (info.path == "") break;
|
||||
|
||||
/* !!! inefficient */
|
||||
for (WeakGoals::iterator k = waitingForInfo.begin();
|
||||
k != waitingForInfo.end(); ++k)
|
||||
{
|
||||
GoalPtr goal = k->lock();
|
||||
if (goal) {
|
||||
SubstitutionGoal * goal2 = dynamic_cast<SubstitutionGoal *>(goal.get());
|
||||
if (goal2->storePath == info.path) {
|
||||
goal2->references = info.references;
|
||||
goal2->deriver = info.deriver;
|
||||
goal2->infoOkay = true;
|
||||
wakeUp(goal);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (WeakGoals::iterator k = waitingForInfo.begin();
|
||||
k != waitingForInfo.end(); ++k)
|
||||
{
|
||||
GoalPtr goal = k->lock();
|
||||
if (goal) wakeUp(goal);
|
||||
}
|
||||
|
||||
requestedInfo.clear();
|
||||
waitingForInfo.clear(); // !!! have we done them all?
|
||||
}
|
||||
|
||||
|
||||
void Worker::run(const Goals & _topGoals)
|
||||
{
|
||||
for (Goals::iterator i = _topGoals.begin();
|
||||
|
|
@ -2711,8 +2614,6 @@ void Worker::run(const Goals & _topGoals)
|
|||
|
||||
if (topGoals.empty()) break;
|
||||
|
||||
getInfo();
|
||||
|
||||
/* Wait for input. */
|
||||
if (!children.empty())
|
||||
waitForInput();
|
||||
|
|
|
|||
|
|
@ -83,6 +83,13 @@ LocalStore::~LocalStore()
|
|||
{
|
||||
try {
|
||||
flushDelayedUpdates();
|
||||
|
||||
foreach (RunningSubstituters::iterator, i, runningSubstituters) {
|
||||
i->second.toBuf.reset();
|
||||
i->second.to.reset();
|
||||
i->second.pid.wait(true);
|
||||
}
|
||||
|
||||
} catch (...) {
|
||||
ignoreException();
|
||||
}
|
||||
|
|
@ -367,8 +374,6 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path)
|
|||
std::map<Path, ValidPathInfo>::iterator lookup = pathInfoCache.find(path);
|
||||
if (lookup != pathInfoCache.end()) return lookup->second;
|
||||
|
||||
//printMsg(lvlError, "queryPathInfo: " + path);
|
||||
|
||||
/* Read the info file. */
|
||||
Path infoFile = infoFileFor(path);
|
||||
if (!pathExists(infoFile))
|
||||
|
|
@ -467,33 +472,105 @@ Path LocalStore::queryDeriver(const Path & path)
|
|||
}
|
||||
|
||||
|
||||
PathSet LocalStore::querySubstitutablePaths()
|
||||
void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & run)
|
||||
{
|
||||
if (!substitutablePathsLoaded) {
|
||||
for (Paths::iterator i = substituters.begin(); i != substituters.end(); ++i) {
|
||||
debug(format("running `%1%' to find out substitutable paths") % *i);
|
||||
Strings args;
|
||||
args.push_back("--query-paths");
|
||||
Strings ss = tokenizeString(runProgram(*i, false, args), "\n");
|
||||
for (Strings::iterator j = ss.begin(); j != ss.end(); ++j) {
|
||||
if (!isStorePath(*j))
|
||||
throw Error(format("`%1%' returned a bad substitutable path `%2%'")
|
||||
% *i % *j);
|
||||
substitutablePaths.insert(*j);
|
||||
}
|
||||
if (run.pid != -1) return;
|
||||
|
||||
debug(format("starting substituter program `%1%'") % substituter);
|
||||
|
||||
Pipe toPipe, fromPipe;
|
||||
|
||||
toPipe.create();
|
||||
fromPipe.create();
|
||||
|
||||
run.pid = fork();
|
||||
|
||||
switch (run.pid) {
|
||||
|
||||
case -1:
|
||||
throw SysError("unable to fork");
|
||||
|
||||
case 0: /* child */
|
||||
try {
|
||||
fromPipe.readSide.close();
|
||||
toPipe.writeSide.close();
|
||||
if (dup2(toPipe.readSide, STDIN_FILENO) == -1)
|
||||
throw SysError("dupping stdin");
|
||||
if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1)
|
||||
throw SysError("dupping stdout");
|
||||
closeMostFDs(set<int>());
|
||||
execl(substituter.c_str(), substituter.c_str(), "--query", NULL);
|
||||
throw SysError(format("executing `%1%'") % substituter);
|
||||
} catch (std::exception & e) {
|
||||
std::cerr << "error: " << e.what() << std::endl;
|
||||
}
|
||||
substitutablePathsLoaded = true;
|
||||
quickExit(1);
|
||||
}
|
||||
|
||||
return substitutablePaths;
|
||||
/* Parent. */
|
||||
|
||||
toPipe.readSide.close();
|
||||
fromPipe.writeSide.close();
|
||||
|
||||
run.toBuf = boost::shared_ptr<stdio_filebuf>(new stdio_filebuf(toPipe.writeSide.borrow(), std::ios_base::out));
|
||||
run.to = boost::shared_ptr<std::ostream>(new std::ostream(&*run.toBuf));
|
||||
|
||||
run.fromBuf = boost::shared_ptr<stdio_filebuf>(new stdio_filebuf(fromPipe.readSide.borrow(), std::ios_base::in));
|
||||
run.from = boost::shared_ptr<std::istream>(new std::istream(&*run.fromBuf));
|
||||
}
|
||||
|
||||
|
||||
bool LocalStore::hasSubstitutes(const Path & path)
|
||||
{
|
||||
if (!substitutablePathsLoaded)
|
||||
querySubstitutablePaths();
|
||||
return substitutablePaths.find(path) != substitutablePaths.end();
|
||||
foreach (Paths::iterator, i, substituters) {
|
||||
RunningSubstituter & run(runningSubstituters[*i]);
|
||||
startSubstituter(*i, run);
|
||||
|
||||
*run.to << "have\n" << path << "\n" << std::flush;
|
||||
|
||||
string s;
|
||||
|
||||
int res;
|
||||
getline(*run.from, s);
|
||||
if (!string2Int(s, res)) abort();
|
||||
|
||||
if (res) return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
bool LocalStore::querySubstitutablePathInfo(const Path & path,
|
||||
SubstitutablePathInfo & info)
|
||||
{
|
||||
foreach (Paths::iterator, i, substituters) {
|
||||
RunningSubstituter & run(runningSubstituters[*i]);
|
||||
startSubstituter(*i, run);
|
||||
|
||||
*run.to << "info\n" << path << "\n" << std::flush;
|
||||
|
||||
string s;
|
||||
|
||||
int res;
|
||||
getline(*run.from, s);
|
||||
if (!string2Int(s, res)) abort();
|
||||
|
||||
if (res) {
|
||||
getline(*run.from, info.deriver);
|
||||
int nrRefs;
|
||||
getline(*run.from, s);
|
||||
if (!string2Int(s, nrRefs)) abort();
|
||||
while (nrRefs--) {
|
||||
Path p; getline(*run.from, p);
|
||||
info.references.insert(p);
|
||||
}
|
||||
info.downloadSize = 0;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@
|
|||
|
||||
#include <string>
|
||||
|
||||
#include <ext/stdio_filebuf.h>
|
||||
|
||||
#include "store-api.hh"
|
||||
#include "util.hh"
|
||||
|
||||
|
|
@ -34,11 +36,26 @@ struct OptimiseStats
|
|||
};
|
||||
|
||||
|
||||
typedef __gnu_cxx::stdio_filebuf<char> stdio_filebuf;
|
||||
|
||||
|
||||
struct RunningSubstituter
|
||||
{
|
||||
Pid pid;
|
||||
boost::shared_ptr<stdio_filebuf> toBuf, fromBuf;
|
||||
boost::shared_ptr<std::ostream> to;
|
||||
boost::shared_ptr<std::istream> from;
|
||||
};
|
||||
|
||||
|
||||
class LocalStore : public StoreAPI
|
||||
{
|
||||
private:
|
||||
bool substitutablePathsLoaded;
|
||||
PathSet substitutablePaths;
|
||||
|
||||
typedef std::map<Path, RunningSubstituter> RunningSubstituters;
|
||||
RunningSubstituters runningSubstituters;
|
||||
|
||||
public:
|
||||
|
||||
|
|
@ -65,6 +82,9 @@ public:
|
|||
PathSet querySubstitutablePaths();
|
||||
|
||||
bool hasSubstitutes(const Path & path);
|
||||
|
||||
bool querySubstitutablePathInfo(const Path & path,
|
||||
SubstitutablePathInfo & info);
|
||||
|
||||
Path addToStore(const Path & srcPath, bool fixed = false,
|
||||
bool recursive = false, string hashAlgo = "",
|
||||
|
|
@ -147,6 +167,9 @@ private:
|
|||
void tryToDelete(const GCOptions & options, GCResults & results,
|
||||
const PathSet & livePaths, const PathSet & tempRootsClosed, PathSet & done,
|
||||
const Path & path);
|
||||
|
||||
void startSubstituter(const Path & substituter,
|
||||
RunningSubstituter & runningSubstituter);
|
||||
};
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
#include "misc.hh"
|
||||
#include "store-api.hh"
|
||||
#include "local-store.hh"
|
||||
|
||||
#include <aterm2.h>
|
||||
|
||||
|
|
@ -79,10 +80,13 @@ void queryMissing(const PathSet & targets,
|
|||
|
||||
else {
|
||||
if (store->isValidPath(p)) continue;
|
||||
if (store->hasSubstitutes(p))
|
||||
SubstitutablePathInfo info;
|
||||
if (dynamic_cast<LocalStore *>(store.get())->querySubstitutablePathInfo(p, info)) {
|
||||
willSubstitute.insert(p);
|
||||
// XXX call the substituters
|
||||
// store->queryReferences(p, todo);
|
||||
todo.insert(info.references.begin(), info.references.end());
|
||||
}
|
||||
/* Not substitutable and not buildable; should we flag
|
||||
this? */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -213,6 +213,13 @@ bool RemoteStore::hasSubstitutes(const Path & path)
|
|||
}
|
||||
|
||||
|
||||
bool RemoteStore::querySubstitutablePathInfo(const Path & path,
|
||||
SubstitutablePathInfo & info)
|
||||
{
|
||||
throw Error("not implemented");
|
||||
}
|
||||
|
||||
|
||||
Hash RemoteStore::queryPathHash(const Path & path)
|
||||
{
|
||||
writeInt(wopQueryPathHash, to);
|
||||
|
|
@ -256,12 +263,6 @@ Path RemoteStore::queryDeriver(const Path & path)
|
|||
}
|
||||
|
||||
|
||||
PathSet RemoteStore::querySubstitutablePaths()
|
||||
{
|
||||
throw Error("not implemented");
|
||||
}
|
||||
|
||||
|
||||
Path RemoteStore::addToStore(const Path & _srcPath, bool fixed,
|
||||
bool recursive, string hashAlgo, PathFilter & filter)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -37,10 +37,11 @@ public:
|
|||
|
||||
Path queryDeriver(const Path & path);
|
||||
|
||||
PathSet querySubstitutablePaths();
|
||||
|
||||
bool hasSubstitutes(const Path & path);
|
||||
|
||||
bool querySubstitutablePathInfo(const Path & path,
|
||||
SubstitutablePathInfo & info);
|
||||
|
||||
Path addToStore(const Path & srcPath, bool fixed = false,
|
||||
bool recursive = false, string hashAlgo = "",
|
||||
PathFilter & filter = defaultPathFilter);
|
||||
|
|
|
|||
|
|
@ -17,13 +17,6 @@ GCOptions::GCOptions()
|
|||
}
|
||||
|
||||
|
||||
bool StoreAPI::hasSubstitutes(const Path & path)
|
||||
{
|
||||
PathSet paths = querySubstitutablePaths();
|
||||
return paths.find(path) != paths.end();
|
||||
}
|
||||
|
||||
|
||||
bool isInStore(const Path & path)
|
||||
{
|
||||
return path[0] == '/'
|
||||
|
|
|
|||
|
|
@ -88,6 +88,14 @@ struct GCResults
|
|||
};
|
||||
|
||||
|
||||
struct SubstitutablePathInfo
|
||||
{
|
||||
Path deriver;
|
||||
PathSet references;
|
||||
unsigned long long downloadSize; /* 0 = unknown or inapplicable */
|
||||
};
|
||||
|
||||
|
||||
class StoreAPI
|
||||
{
|
||||
public:
|
||||
|
|
@ -117,12 +125,13 @@ public:
|
|||
no deriver has been set. */
|
||||
virtual Path queryDeriver(const Path & path) = 0;
|
||||
|
||||
/* Query the set of substitutable paths. */
|
||||
virtual PathSet querySubstitutablePaths() = 0;
|
||||
/* Query whether a path has substitutes. */
|
||||
virtual bool hasSubstitutes(const Path & path) = 0;
|
||||
|
||||
/* More efficient variant if we just want to know if a path has
|
||||
substitutes. */
|
||||
virtual bool hasSubstitutes(const Path & path);
|
||||
/* Query the references, deriver and download size of a
|
||||
substitutable path. */
|
||||
virtual bool querySubstitutablePathInfo(const Path & path,
|
||||
SubstitutablePathInfo & info) = 0;
|
||||
|
||||
/* Copy the contents of a path to the store and register the
|
||||
validity the resulting path. The resulting path is returned.
|
||||
|
|
|
|||
|
|
@ -579,7 +579,14 @@ AutoCloseFD::AutoCloseFD(int fd)
|
|||
|
||||
AutoCloseFD::AutoCloseFD(const AutoCloseFD & fd)
|
||||
{
|
||||
abort();
|
||||
/* Copying a AutoCloseFD isn't allowed (who should get to close
|
||||
it?). But as a edge case, allow copying of closed
|
||||
AutoCloseFDs. This is necessary due to tiresome reasons
|
||||
involving copy constructor use on default object values in STL
|
||||
containers (like when you do `map[value]' where value isn't in
|
||||
the map yet). */
|
||||
this->fd = fd.fd;
|
||||
if (this->fd != -1) abort();
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -832,7 +839,7 @@ string runProgram(Path program, bool searchPath, const Strings & args)
|
|||
pipe.readSide.close();
|
||||
|
||||
if (dup2(pipe.writeSide, STDOUT_FILENO) == -1)
|
||||
throw SysError("dupping from-hook write side");
|
||||
throw SysError("dupping stdout");
|
||||
|
||||
std::vector<const char *> cargs; /* careful with c_str()! */
|
||||
cargs.push_back(program.c_str());
|
||||
|
|
@ -868,6 +875,17 @@ string runProgram(Path program, bool searchPath, const Strings & args)
|
|||
}
|
||||
|
||||
|
||||
void closeMostFDs(const set<int> & exceptions)
|
||||
{
|
||||
int maxFD = 0;
|
||||
maxFD = sysconf(_SC_OPEN_MAX);
|
||||
for (int fd = 0; fd < maxFD; ++fd)
|
||||
if (fd != STDIN_FILENO && fd != STDOUT_FILENO && fd != STDERR_FILENO
|
||||
&& exceptions.find(fd) == exceptions.end())
|
||||
close(fd); /* ignore result */
|
||||
}
|
||||
|
||||
|
||||
void quickExit(int status)
|
||||
{
|
||||
#ifdef __CYGWIN__
|
||||
|
|
|
|||
|
|
@ -245,6 +245,10 @@ void killUser(uid_t uid);
|
|||
string runProgram(Path program, bool searchPath = false,
|
||||
const Strings & args = Strings());
|
||||
|
||||
/* Close all file descriptors except stdin, stdout, stderr, and those
|
||||
listed in the given set. Good practice in child processes. */
|
||||
void closeMostFDs(const set<int> & exceptions);
|
||||
|
||||
/* Wrapper around _exit() on Unix and ExitProcess() on Windows. (On
|
||||
Cygwin, _exit() doesn't seem to do the right thing.) */
|
||||
void quickExit(int status);
|
||||
|
|
|
|||
|
|
@ -1,14 +1,25 @@
|
|||
#! /bin/sh -e
|
||||
echo substituter args: $* >&2
|
||||
|
||||
if test $1 = "--query-paths"; then
|
||||
cat $TEST_ROOT/sub-paths
|
||||
elif test $1 = "--query-info"; then
|
||||
shift
|
||||
for i in in $@; do
|
||||
echo $i
|
||||
echo "" # deriver
|
||||
echo 0 # nr of refs
|
||||
if test $1 = "--query"; then
|
||||
while read cmd; do
|
||||
echo FOO $cmd >&2
|
||||
if test "$cmd" = "have"; then
|
||||
read path
|
||||
if grep -q "$path" $TEST_ROOT/sub-paths; then
|
||||
echo 1
|
||||
else
|
||||
echo 0
|
||||
fi
|
||||
elif test "$cmd" = "info"; then
|
||||
read path
|
||||
echo 1
|
||||
echo "" # deriver
|
||||
echo 0 # nr of refs
|
||||
else
|
||||
echo "bad command $cmd"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
elif test $1 = "--substitute"; then
|
||||
mkdir $2
|
||||
|
|
|
|||
|
|
@ -1,14 +1,24 @@
|
|||
#! /bin/sh -e
|
||||
echo substituter2 args: $* >&2
|
||||
|
||||
if test $1 = "--query-paths"; then
|
||||
cat $TEST_ROOT/sub-paths
|
||||
elif test $1 = "--query-info"; then
|
||||
shift
|
||||
for i in in $@; do
|
||||
echo $i
|
||||
echo "" # deriver
|
||||
echo 0 # nr of refs
|
||||
if test $1 = "--query"; then
|
||||
while read cmd; do
|
||||
if test "$cmd" = "have"; then
|
||||
read path
|
||||
if grep -q "$path" $TEST_ROOT/sub-paths; then
|
||||
echo 1
|
||||
else
|
||||
echo 0
|
||||
fi
|
||||
elif test "$cmd" = "info"; then
|
||||
read path
|
||||
echo 1
|
||||
echo "" # deriver
|
||||
echo 0 # nr of refs
|
||||
else
|
||||
echo "bad command $cmd"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
elif test $1 = "--substitute"; then
|
||||
exit 1
|
||||
|
|
|
|||
Loading…
Reference in a new issue