LocalStore: Allow the physical and logical store directories to differ
This is primarily to subsume the functionality of the copy-from-other-stores substituter. For example, in the NixOS installer, we can now do (assuming we're in the target chroot, and the Nix store of the installation CD is bind-mounted on /tmp/nix): $ nix-build ... --option substituters 'local?state=/tmp/nix/var&real=/tmp/nix/store' However, unlike copy-from-other-stores, this also allows write access to such a store. One application might be fetching substitutes for /nix/store in a situation where the user doesn't have sufficient privileges to create /nix, e.g.: $ NIX_REMOTE="local?state=/home/alice/nix/var&real=/home/alice/nix/store" nix-build ...
This commit is contained in:
Eelco Dolstra
parent
064816ab98
commit
4494000e04
7 changed files with 68 additions and 158 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -44,7 +44,6 @@ Makefile.config
|
|||
/scripts/nix-copy-closure
|
||||
/scripts/NixConfig.pm
|
||||
/scripts/NixManifest.pm
|
||||
/scripts/copy-from-other-stores.pl
|
||||
/scripts/download-from-binary-cache.pl
|
||||
/scripts/find-runtime-roots.pl
|
||||
/scripts/build-remote.pl
|
||||
|
|
|
|||
|
|
@ -1,103 +0,0 @@
|
|||
#! @perl@ -w @perlFlags@
|
||||
|
||||
use utf8;
|
||||
use strict;
|
||||
use File::Basename;
|
||||
use IO::Handle;
|
||||
|
||||
my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
|
||||
|
||||
|
||||
STDOUT->autoflush(1);
|
||||
binmode STDERR, ":encoding(utf8)";
|
||||
|
||||
my @remoteStoresAll = split ':', ($ENV{"NIX_OTHER_STORES"} or "");
|
||||
|
||||
my @remoteStores;
|
||||
foreach my $dir (@remoteStoresAll) {
|
||||
push @remoteStores, glob($dir);
|
||||
}
|
||||
|
||||
exit if scalar @remoteStores == 0;
|
||||
print "\n";
|
||||
|
||||
|
||||
$ENV{"NIX_REMOTE"} = "";
|
||||
|
||||
|
||||
sub findStorePath {
|
||||
my $storePath = shift;
|
||||
foreach my $store (@remoteStores) {
|
||||
my $sourcePath = "$store/store/" . basename $storePath;
|
||||
next unless -e $sourcePath || -l $sourcePath;
|
||||
$ENV{"NIX_DB_DIR"} = "$store/var/nix/db";
|
||||
return ($store, $sourcePath) if
|
||||
system("$binDir/nix-store --check-validity $storePath") == 0;
|
||||
}
|
||||
return undef;
|
||||
}
|
||||
|
||||
|
||||
if ($ARGV[0] eq "--query") {
|
||||
|
||||
while (<STDIN>) {
|
||||
chomp;
|
||||
my ($cmd, @args) = split " ", $_;
|
||||
|
||||
if ($cmd eq "have") {
|
||||
foreach my $storePath (@args) {
|
||||
print "$storePath\n" if defined findStorePath($storePath);
|
||||
}
|
||||
print "\n";
|
||||
}
|
||||
|
||||
elsif ($cmd eq "info") {
|
||||
foreach my $storePath (@args) {
|
||||
my ($store, $sourcePath) = findStorePath($storePath);
|
||||
next unless defined $store;
|
||||
|
||||
$ENV{"NIX_DB_DIR"} = "$store/var/nix/db";
|
||||
|
||||
my $deriver = `$binDir/nix-store --query --deriver $storePath`;
|
||||
die "cannot query deriver of ‘$storePath’" if $? != 0;
|
||||
chomp $deriver;
|
||||
$deriver = "" if $deriver eq "unknown-deriver";
|
||||
|
||||
my @references = split "\n",
|
||||
`$binDir/nix-store --query --references $storePath`;
|
||||
die "cannot query references of ‘$storePath’" if $? != 0;
|
||||
|
||||
my $narSize = `$binDir/nix-store --query --size $storePath`;
|
||||
die "cannot query size of ‘$storePath’" if $? != 0;
|
||||
chomp $narSize;
|
||||
|
||||
print "$storePath\n";
|
||||
print "$deriver\n";
|
||||
print scalar @references, "\n";
|
||||
print "$_\n" foreach @references;
|
||||
print "0\n";
|
||||
print "$narSize\n";
|
||||
}
|
||||
|
||||
print "\n";
|
||||
}
|
||||
|
||||
else { die "unknown command ‘$cmd’"; }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
elsif ($ARGV[0] eq "--substitute") {
|
||||
die unless scalar @ARGV == 3;
|
||||
my $storePath = $ARGV[1];
|
||||
my $destPath = $ARGV[2];
|
||||
my ($store, $sourcePath) = findStorePath $storePath;
|
||||
die unless $store;
|
||||
print STDERR "\n*** Copying ‘$storePath’ from ‘$sourcePath’\n\n";
|
||||
system("@coreutils@/cp", "-rpd", $sourcePath, $destPath) == 0
|
||||
or die "cannot copy ‘$sourcePath’ to ‘$storePath’";
|
||||
print "\n"; # no hash to verify
|
||||
}
|
||||
|
||||
|
||||
else { die; }
|
||||
|
|
@ -1269,6 +1269,9 @@ void DerivationGoal::tryToBuild()
|
|||
{
|
||||
trace("trying to build");
|
||||
|
||||
if (worker.store.storeDir != worker.store.realStoreDir)
|
||||
throw Error("building with a diverted Nix store is not supported");
|
||||
|
||||
/* Check for the possibility that some other goal in this process
|
||||
has locked the output since we checked in haveDerivation().
|
||||
(It can't happen between here and the lockPaths() call below
|
||||
|
|
|
|||
|
|
@ -370,7 +370,6 @@ struct LocalStore::GCState
|
|||
bool gcKeepDerivations;
|
||||
unsigned long long bytesInvalidated;
|
||||
bool moveToTrash = true;
|
||||
Path trashDir;
|
||||
bool shouldDelete;
|
||||
GCState(GCResults & results_) : results(results_), bytesInvalidated(0) { }
|
||||
};
|
||||
|
|
@ -407,10 +406,12 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
|
|||
invalidatePathChecked(path);
|
||||
}
|
||||
|
||||
Path realPath = realStoreDir + "/" + baseNameOf(path);
|
||||
|
||||
struct stat st;
|
||||
if (lstat(path.c_str(), &st)) {
|
||||
if (lstat(realPath.c_str(), &st)) {
|
||||
if (errno == ENOENT) return;
|
||||
throw SysError(format("getting status of %1%") % path);
|
||||
throw SysError(format("getting status of %1%") % realPath);
|
||||
}
|
||||
|
||||
printMsg(lvlInfo, format("deleting ‘%1%’") % path);
|
||||
|
|
@ -427,20 +428,20 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
|
|||
// if the path was not valid, need to determine the actual
|
||||
// size.
|
||||
try {
|
||||
if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
|
||||
throw SysError(format("making ‘%1%’ writable") % path);
|
||||
Path tmp = state.trashDir + "/" + baseNameOf(path);
|
||||
if (rename(path.c_str(), tmp.c_str()))
|
||||
throw SysError(format("unable to rename ‘%1%’ to ‘%2%’") % path % tmp);
|
||||
if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1)
|
||||
throw SysError(format("making ‘%1%’ writable") % realPath);
|
||||
Path tmp = trashDir + "/" + baseNameOf(path);
|
||||
if (rename(realPath.c_str(), tmp.c_str()))
|
||||
throw SysError(format("unable to rename ‘%1%’ to ‘%2%’") % realPath % tmp);
|
||||
state.bytesInvalidated += size;
|
||||
} catch (SysError & e) {
|
||||
if (e.errNo == ENOSPC) {
|
||||
printMsg(lvlInfo, format("note: can't create move ‘%1%’: %2%") % path % e.msg());
|
||||
deleteGarbage(state, path);
|
||||
printMsg(lvlInfo, format("note: can't create move ‘%1%’: %2%") % realPath % e.msg());
|
||||
deleteGarbage(state, realPath);
|
||||
}
|
||||
}
|
||||
} else
|
||||
deleteGarbage(state, path);
|
||||
deleteGarbage(state, realPath);
|
||||
|
||||
if (state.results.bytesFreed + state.bytesInvalidated > state.options.maxFreed) {
|
||||
printMsg(lvlInfo, format("deleted or invalidated more than %1% bytes; stopping") % state.options.maxFreed);
|
||||
|
|
@ -508,7 +509,8 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
|
|||
{
|
||||
checkInterrupt();
|
||||
|
||||
if (path == linksDir || path == state.trashDir) return;
|
||||
auto realPath = realStoreDir + "/" + baseNameOf(path);
|
||||
if (realPath == linksDir || realPath == trashDir) return;
|
||||
|
||||
Activity act(*logger, lvlDebug, format("considering whether to delete ‘%1%’") % path);
|
||||
|
||||
|
|
@ -590,7 +592,6 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
|||
{
|
||||
GCState state(results);
|
||||
state.options = options;
|
||||
state.trashDir = storeDir + "/trash";
|
||||
state.gcKeepOutputs = settings.gcKeepOutputs;
|
||||
state.gcKeepDerivations = settings.gcKeepDerivations;
|
||||
|
||||
|
|
@ -639,9 +640,9 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
|||
that is not reachable from `roots' is garbage. */
|
||||
|
||||
if (state.shouldDelete) {
|
||||
if (pathExists(state.trashDir)) deleteGarbage(state, state.trashDir);
|
||||
if (pathExists(trashDir)) deleteGarbage(state, trashDir);
|
||||
try {
|
||||
createDirs(state.trashDir);
|
||||
createDirs(trashDir);
|
||||
} catch (SysError & e) {
|
||||
if (e.errNo == ENOSPC) {
|
||||
printMsg(lvlInfo, format("note: can't create trash directory: %1%") % e.msg());
|
||||
|
|
@ -671,8 +672,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
|||
|
||||
try {
|
||||
|
||||
AutoCloseDir dir = opendir(storeDir.c_str());
|
||||
if (!dir) throw SysError(format("opening directory ‘%1%’") % storeDir);
|
||||
AutoCloseDir dir = opendir(realStoreDir.c_str());
|
||||
if (!dir) throw SysError(format("opening directory ‘%1%’") % realStoreDir);
|
||||
|
||||
/* Read the store and immediately delete all paths that
|
||||
aren't valid. When using --max-freed etc., deleting
|
||||
|
|
@ -725,8 +726,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
|||
fds.clear();
|
||||
|
||||
/* Delete the trash directory. */
|
||||
printMsg(lvlInfo, format("deleting ‘%1%’") % state.trashDir);
|
||||
deleteGarbage(state, state.trashDir);
|
||||
printMsg(lvlInfo, format("deleting ‘%1%’") % trashDir);
|
||||
deleteGarbage(state, trashDir);
|
||||
|
||||
/* Clean up the links directory. */
|
||||
if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) {
|
||||
|
|
|
|||
|
|
@ -38,10 +38,12 @@ namespace nix {
|
|||
|
||||
LocalStore::LocalStore(const Params & params)
|
||||
: LocalFSStore(params)
|
||||
, realStoreDir(get(params, "real", storeDir))
|
||||
, dbDir(get(params, "state", "") != "" ? get(params, "state", "") + "/db" : settings.nixDBPath)
|
||||
, linksDir(storeDir + "/.links")
|
||||
, linksDir(realStoreDir + "/.links")
|
||||
, reservedPath(dbDir + "/reserved")
|
||||
, schemaPath(dbDir + "/schema")
|
||||
, trashDir(realStoreDir + "/trash")
|
||||
, requireSigs(settings.get("signed-binary-caches", std::string("")) != "") // FIXME: rename option
|
||||
, publicKeys(getDefaultPublicKeys())
|
||||
{
|
||||
|
|
@ -53,7 +55,7 @@ LocalStore::LocalStore(const Params & params)
|
|||
}
|
||||
|
||||
/* Create missing state directories if they don't already exist. */
|
||||
createDirs(storeDir);
|
||||
createDirs(realStoreDir);
|
||||
makeStoreWritable();
|
||||
createDirs(linksDir);
|
||||
Path profilesDir = stateDir + "/profiles";
|
||||
|
|
@ -83,21 +85,21 @@ LocalStore::LocalStore(const Params & params)
|
|||
% settings.buildUsersGroup);
|
||||
else {
|
||||
struct stat st;
|
||||
if (stat(storeDir.c_str(), &st))
|
||||
throw SysError(format("getting attributes of path ‘%1%’") % storeDir);
|
||||
if (stat(realStoreDir.c_str(), &st))
|
||||
throw SysError(format("getting attributes of path ‘%1%’") % realStoreDir);
|
||||
|
||||
if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) {
|
||||
if (chown(storeDir.c_str(), 0, gr->gr_gid) == -1)
|
||||
throw SysError(format("changing ownership of path ‘%1%’") % storeDir);
|
||||
if (chmod(storeDir.c_str(), perm) == -1)
|
||||
throw SysError(format("changing permissions on path ‘%1%’") % storeDir);
|
||||
if (chown(realStoreDir.c_str(), 0, gr->gr_gid) == -1)
|
||||
throw SysError(format("changing ownership of path ‘%1%’") % realStoreDir);
|
||||
if (chmod(realStoreDir.c_str(), perm) == -1)
|
||||
throw SysError(format("changing permissions on path ‘%1%’") % realStoreDir);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Ensure that the store and its parents are not symlinks. */
|
||||
if (getEnv("NIX_IGNORE_SYMLINK_STORE") != "1") {
|
||||
Path path = storeDir;
|
||||
Path path = realStoreDir;
|
||||
struct stat st;
|
||||
while (path != "/") {
|
||||
if (lstat(path.c_str(), &st))
|
||||
|
|
@ -343,15 +345,15 @@ void LocalStore::makeStoreWritable()
|
|||
if (getuid() != 0) return;
|
||||
/* Check if /nix/store is on a read-only mount. */
|
||||
struct statvfs stat;
|
||||
if (statvfs(storeDir.c_str(), &stat) != 0)
|
||||
if (statvfs(realStoreDir.c_str(), &stat) != 0)
|
||||
throw SysError("getting info about the Nix store mount point");
|
||||
|
||||
if (stat.f_flag & ST_RDONLY) {
|
||||
if (unshare(CLONE_NEWNS) == -1)
|
||||
throw SysError("setting up a private mount namespace");
|
||||
|
||||
if (mount(0, storeDir.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1)
|
||||
throw SysError(format("remounting %1% writable") % storeDir);
|
||||
if (mount(0, realStoreDir.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1)
|
||||
throw SysError(format("remounting %1% writable") % realStoreDir);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
|
@ -917,23 +919,25 @@ void LocalStore::addToStore(const ValidPathInfo & info, const std::string & nar,
|
|||
|
||||
PathLocks outputLock;
|
||||
|
||||
Path realPath = realStoreDir + "/" + baseNameOf(info.path);
|
||||
|
||||
/* Lock the output path. But don't lock if we're being called
|
||||
from a build hook (whose parent process already acquired a
|
||||
lock on this path). */
|
||||
Strings locksHeld = tokenizeString<Strings>(getEnv("NIX_HELD_LOCKS"));
|
||||
if (find(locksHeld.begin(), locksHeld.end(), info.path) == locksHeld.end())
|
||||
outputLock.lockPaths({info.path});
|
||||
outputLock.lockPaths({realPath});
|
||||
|
||||
if (repair || !isValidPath(info.path)) {
|
||||
|
||||
deletePath(info.path);
|
||||
deletePath(realPath);
|
||||
|
||||
StringSource source(nar);
|
||||
restorePath(info.path, source);
|
||||
restorePath(realPath, source);
|
||||
|
||||
canonicalisePathMetaData(info.path, -1);
|
||||
canonicalisePathMetaData(realPath, -1);
|
||||
|
||||
optimisePath(info.path); // FIXME: combine with hashPath()
|
||||
optimisePath(realPath); // FIXME: combine with hashPath()
|
||||
|
||||
registerValidPath(info);
|
||||
}
|
||||
|
|
@ -957,19 +961,21 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
|
|||
/* The first check above is an optimisation to prevent
|
||||
unnecessary lock acquisition. */
|
||||
|
||||
PathLocks outputLock({dstPath});
|
||||
Path realPath = realStoreDir + "/" + baseNameOf(dstPath);
|
||||
|
||||
PathLocks outputLock({realPath});
|
||||
|
||||
if (repair || !isValidPath(dstPath)) {
|
||||
|
||||
deletePath(dstPath);
|
||||
deletePath(realPath);
|
||||
|
||||
if (recursive) {
|
||||
StringSource source(dump);
|
||||
restorePath(dstPath, source);
|
||||
restorePath(realPath, source);
|
||||
} else
|
||||
writeFile(dstPath, dump);
|
||||
writeFile(realPath, dump);
|
||||
|
||||
canonicalisePathMetaData(dstPath, -1);
|
||||
canonicalisePathMetaData(realPath, -1);
|
||||
|
||||
/* Register the SHA-256 hash of the NAR serialisation of
|
||||
the path in the database. We may just have computed it
|
||||
|
|
@ -980,9 +986,9 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
|
|||
hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump);
|
||||
hash.second = dump.size();
|
||||
} else
|
||||
hash = hashPath(htSHA256, dstPath);
|
||||
hash = hashPath(htSHA256, realPath);
|
||||
|
||||
optimisePath(dstPath); // FIXME: combine with hashPath()
|
||||
optimisePath(realPath); // FIXME: combine with hashPath()
|
||||
|
||||
ValidPathInfo info;
|
||||
info.path = dstPath;
|
||||
|
|
@ -1026,21 +1032,23 @@ Path LocalStore::addTextToStore(const string & name, const string & s,
|
|||
|
||||
if (repair || !isValidPath(dstPath)) {
|
||||
|
||||
PathLocks outputLock({dstPath});
|
||||
Path realPath = realStoreDir + "/" + baseNameOf(dstPath);
|
||||
|
||||
PathLocks outputLock({realPath});
|
||||
|
||||
if (repair || !isValidPath(dstPath)) {
|
||||
|
||||
deletePath(dstPath);
|
||||
deletePath(realPath);
|
||||
|
||||
writeFile(dstPath, s);
|
||||
writeFile(realPath, s);
|
||||
|
||||
canonicalisePathMetaData(dstPath, -1);
|
||||
canonicalisePathMetaData(realPath, -1);
|
||||
|
||||
StringSink sink;
|
||||
dumpString(s, sink);
|
||||
auto hash = hashString(htSHA256, *sink.s);
|
||||
|
||||
optimisePath(dstPath);
|
||||
optimisePath(realPath);
|
||||
|
||||
ValidPathInfo info;
|
||||
info.path = dstPath;
|
||||
|
|
@ -1067,7 +1075,7 @@ Path LocalStore::createTempDirInStore()
|
|||
/* There is a slight possibility that `tmpDir' gets deleted by
|
||||
the GC between createTempDir() and addTempRoot(), so repeat
|
||||
until `tmpDir' exists. */
|
||||
tmpDir = createTempDir(storeDir);
|
||||
tmpDir = createTempDir(realStoreDir);
|
||||
addTempRoot(tmpDir);
|
||||
} while (!pathExists(tmpDir));
|
||||
return tmpDir;
|
||||
|
|
@ -1107,7 +1115,7 @@ bool LocalStore::verifyStore(bool checkContents, bool repair)
|
|||
AutoCloseFD fdGCLock = openGCLock(ltWrite);
|
||||
|
||||
PathSet store;
|
||||
for (auto & i : readDirectory(storeDir)) store.insert(i.name);
|
||||
for (auto & i : readDirectory(realStoreDir)) store.insert(i.name);
|
||||
|
||||
/* Check whether all valid paths actually exist. */
|
||||
printMsg(lvlInfo, "checking path existence...");
|
||||
|
|
@ -1271,7 +1279,7 @@ void LocalStore::upgradeStore7()
|
|||
{
|
||||
if (getuid() != 0) return;
|
||||
printMsg(lvlError, "removing immutable bits from the Nix store (this may take a while)...");
|
||||
makeMutable(storeDir);
|
||||
makeMutable(realStoreDir);
|
||||
}
|
||||
|
||||
#else
|
||||
|
|
|
|||
|
|
@ -73,10 +73,12 @@ private:
|
|||
|
||||
Sync<State, std::recursive_mutex> _state;
|
||||
|
||||
const Path realStoreDir;
|
||||
const Path dbDir;
|
||||
const Path linksDir;
|
||||
const Path reservedPath;
|
||||
const Path schemaPath;
|
||||
const Path trashDir;
|
||||
|
||||
bool requireSigs;
|
||||
|
||||
|
|
|
|||
|
|
@ -176,7 +176,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa
|
|||
/* Make the containing directory writable, but only if it's not
|
||||
the store itself (we don't want or need to mess with its
|
||||
permissions). */
|
||||
bool mustToggle = !isStorePath(path);
|
||||
bool mustToggle = dirOf(path) != realStoreDir;
|
||||
if (mustToggle) makeWritable(dirOf(path));
|
||||
|
||||
/* When we're done, make the directory read-only again and reset
|
||||
|
|
@ -184,7 +184,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa
|
|||
MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");
|
||||
|
||||
Path tempLink = (format("%1%/.tmp-link-%2%-%3%")
|
||||
% storeDir % getpid() % rand()).str();
|
||||
% realStoreDir % getpid() % rand()).str();
|
||||
|
||||
if (link(linkPath.c_str(), tempLink.c_str()) == -1) {
|
||||
if (errno == EMLINK) {
|
||||
|
|
@ -229,7 +229,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
|
|||
addTempRoot(i);
|
||||
if (!isValidPath(i)) continue; /* path was GC'ed, probably */
|
||||
Activity act(*logger, lvlChatty, format("hashing files in ‘%1%’") % i);
|
||||
optimisePath_(stats, i, inodeHash);
|
||||
optimisePath_(stats, realStoreDir + "/" + baseNameOf(i), inodeHash);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue