Make LocalStore thread-safe
Necessary for multi-threaded commands like "nix verify-paths".
This commit is contained in:
Eelco Dolstra
parent
05fbc606fc
commit
f398949b40
5 changed files with 246 additions and 217 deletions
|
|
@ -239,6 +239,9 @@ private:
|
|||
/* Last time the goals in `waitingForAWhile' where woken up. */
|
||||
time_t lastWokenUp;
|
||||
|
||||
/* Cache for pathContentsGood(). */
|
||||
std::map<Path, bool> pathContentsGoodCache;
|
||||
|
||||
public:
|
||||
|
||||
/* Set if at least one derivation had a BuildError (i.e. permanent
|
||||
|
|
@ -304,6 +307,12 @@ public:
|
|||
void waitForInput();
|
||||
|
||||
unsigned int exitStatus();
|
||||
|
||||
/* Check whether the given valid path exists and has the right
|
||||
contents. */
|
||||
bool pathContentsGood(const Path & path);
|
||||
|
||||
void markContentsGood(const Path & path);
|
||||
};
|
||||
|
||||
|
||||
|
|
@ -1159,7 +1168,7 @@ void DerivationGoal::repairClosure()
|
|||
/* Check each path (slow!). */
|
||||
PathSet broken;
|
||||
for (auto & i : outputClosure) {
|
||||
if (worker.store.pathContentsGood(i)) continue;
|
||||
if (worker.pathContentsGood(i)) continue;
|
||||
printMsg(lvlError, format("found corrupted or missing path ‘%1%’ in the output closure of ‘%2%’") % i % drvPath);
|
||||
Path drvPath2 = outputsToDrv[i];
|
||||
if (drvPath2 == "")
|
||||
|
|
@ -2799,7 +2808,7 @@ void DerivationGoal::registerOutputs()
|
|||
if (curRound == nrRounds) {
|
||||
worker.store.optimisePath(path); // FIXME: combine with scanForReferences()
|
||||
|
||||
worker.store.markContentsGood(path);
|
||||
worker.markContentsGood(path);
|
||||
}
|
||||
|
||||
ValidPathInfo info;
|
||||
|
|
@ -2977,7 +2986,7 @@ PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash)
|
|||
if (!wantOutput(i.first, wantedOutputs)) continue;
|
||||
bool good =
|
||||
worker.store.isValidPath(i.second.path) &&
|
||||
(!checkHash || worker.store.pathContentsGood(i.second.path));
|
||||
(!checkHash || worker.pathContentsGood(i.second.path));
|
||||
if (good == returnValid) result.insert(i.second.path);
|
||||
}
|
||||
return result;
|
||||
|
|
@ -3385,7 +3394,7 @@ void SubstitutionGoal::finished()
|
|||
outputLock->setDeletion(true);
|
||||
outputLock.reset();
|
||||
|
||||
worker.store.markContentsGood(storePath);
|
||||
worker.markContentsGood(storePath);
|
||||
|
||||
printMsg(lvlChatty,
|
||||
format("substitution of path ‘%1%’ succeeded") % storePath);
|
||||
|
|
@ -3785,6 +3794,32 @@ unsigned int Worker::exitStatus()
|
|||
}
|
||||
|
||||
|
||||
bool Worker::pathContentsGood(const Path & path)
|
||||
{
|
||||
std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path);
|
||||
if (i != pathContentsGoodCache.end()) return i->second;
|
||||
printMsg(lvlInfo, format("checking path ‘%1%’...") % path);
|
||||
ValidPathInfo info = store.queryPathInfo(path);
|
||||
bool res;
|
||||
if (!pathExists(path))
|
||||
res = false;
|
||||
else {
|
||||
HashResult current = hashPath(info.narHash.type, path);
|
||||
Hash nullHash(htSHA256);
|
||||
res = info.narHash == nullHash || info.narHash == current.first;
|
||||
}
|
||||
pathContentsGoodCache[path] = res;
|
||||
if (!res) printMsg(lvlError, format("path ‘%1%’ is corrupted or missing!") % path);
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
void Worker::markContentsGood(const Path & path)
|
||||
{
|
||||
pathContentsGoodCache[path] = true;
|
||||
}
|
||||
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -147,35 +147,36 @@ Path Store::addPermRoot(const Path & _storePath,
|
|||
|
||||
void LocalStore::addTempRoot(const Path & path)
|
||||
{
|
||||
auto state(_state.lock());
|
||||
|
||||
/* Create the temporary roots file for this process. */
|
||||
if (fdTempRoots == -1) {
|
||||
if (state->fdTempRoots == -1) {
|
||||
|
||||
while (1) {
|
||||
Path dir = (format("%1%/%2%") % settings.nixStateDir % tempRootsDir).str();
|
||||
createDirs(dir);
|
||||
|
||||
fnTempRoots = (format("%1%/%2%")
|
||||
% dir % getpid()).str();
|
||||
state->fnTempRoots = (format("%1%/%2%") % dir % getpid()).str();
|
||||
|
||||
AutoCloseFD fdGCLock = openGCLock(ltRead);
|
||||
|
||||
if (pathExists(fnTempRoots))
|
||||
if (pathExists(state->fnTempRoots))
|
||||
/* It *must* be stale, since there can be no two
|
||||
processes with the same pid. */
|
||||
unlink(fnTempRoots.c_str());
|
||||
unlink(state->fnTempRoots.c_str());
|
||||
|
||||
fdTempRoots = openLockFile(fnTempRoots, true);
|
||||
state->fdTempRoots = openLockFile(state->fnTempRoots, true);
|
||||
|
||||
fdGCLock.close();
|
||||
|
||||
debug(format("acquiring read lock on ‘%1%’") % fnTempRoots);
|
||||
lockFile(fdTempRoots, ltRead, true);
|
||||
debug(format("acquiring read lock on ‘%1%’") % state->fnTempRoots);
|
||||
lockFile(state->fdTempRoots, ltRead, true);
|
||||
|
||||
/* Check whether the garbage collector didn't get in our
|
||||
way. */
|
||||
struct stat st;
|
||||
if (fstat(fdTempRoots, &st) == -1)
|
||||
throw SysError(format("statting ‘%1%’") % fnTempRoots);
|
||||
if (fstat(state->fdTempRoots, &st) == -1)
|
||||
throw SysError(format("statting ‘%1%’") % state->fnTempRoots);
|
||||
if (st.st_size == 0) break;
|
||||
|
||||
/* The garbage collector deleted this file before we could
|
||||
|
|
@ -187,15 +188,15 @@ void LocalStore::addTempRoot(const Path & path)
|
|||
|
||||
/* Upgrade the lock to a write lock. This will cause us to block
|
||||
if the garbage collector is holding our lock. */
|
||||
debug(format("acquiring write lock on ‘%1%’") % fnTempRoots);
|
||||
lockFile(fdTempRoots, ltWrite, true);
|
||||
debug(format("acquiring write lock on ‘%1%’") % state->fnTempRoots);
|
||||
lockFile(state->fdTempRoots, ltWrite, true);
|
||||
|
||||
string s = path + '\0';
|
||||
writeFull(fdTempRoots, s);
|
||||
writeFull(state->fdTempRoots, s);
|
||||
|
||||
/* Downgrade to a read lock. */
|
||||
debug(format("downgrading to read lock on ‘%1%’") % fnTempRoots);
|
||||
lockFile(fdTempRoots, ltRead, true);
|
||||
debug(format("downgrading to read lock on ‘%1%’") % state->fnTempRoots);
|
||||
lockFile(state->fdTempRoots, ltRead, true);
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -55,20 +55,21 @@ void checkStoreNotSymlink()
|
|||
|
||||
|
||||
LocalStore::LocalStore()
|
||||
: reservedPath(settings.nixDBPath + "/reserved")
|
||||
, didSetSubstituterEnv(false)
|
||||
: linksDir(settings.nixStore + "/.links")
|
||||
, reservedPath(settings.nixDBPath + "/reserved")
|
||||
, schemaPath(settings.nixDBPath + "/schema")
|
||||
{
|
||||
schemaPath = settings.nixDBPath + "/schema";
|
||||
auto state(_state.lock());
|
||||
|
||||
if (settings.readOnlyMode) {
|
||||
openDB(false);
|
||||
openDB(*state, false);
|
||||
return;
|
||||
}
|
||||
|
||||
/* Create missing state directories if they don't already exist. */
|
||||
createDirs(settings.nixStore);
|
||||
makeStoreWritable();
|
||||
createDirs(linksDir = settings.nixStore + "/.links");
|
||||
createDirs(linksDir);
|
||||
Path profilesDir = settings.nixStateDir + "/profiles";
|
||||
createDirs(profilesDir);
|
||||
createDirs(settings.nixStateDir + "/temproots");
|
||||
|
|
@ -140,7 +141,7 @@ LocalStore::LocalStore()
|
|||
} catch (SysError & e) {
|
||||
if (e.errNo != EACCES) throw;
|
||||
settings.readOnlyMode = true;
|
||||
openDB(false);
|
||||
openDB(*state, false);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -158,7 +159,7 @@ LocalStore::LocalStore()
|
|||
|
||||
else if (curSchema == 0) { /* new store */
|
||||
curSchema = nixSchemaVersion;
|
||||
openDB(true);
|
||||
openDB(*state, true);
|
||||
writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str());
|
||||
}
|
||||
|
||||
|
|
@ -186,14 +187,14 @@ LocalStore::LocalStore()
|
|||
|
||||
if (curSchema < 7) { upgradeStore7(); }
|
||||
|
||||
openDB(false);
|
||||
openDB(*state, false);
|
||||
|
||||
if (curSchema < 8) {
|
||||
SQLiteTxn txn(db);
|
||||
if (sqlite3_exec(db, "alter table ValidPaths add column ultimate integer", 0, 0, 0) != SQLITE_OK)
|
||||
throwSQLiteError(db, "upgrading database schema");
|
||||
if (sqlite3_exec(db, "alter table ValidPaths add column sigs text", 0, 0, 0) != SQLITE_OK)
|
||||
throwSQLiteError(db, "upgrading database schema");
|
||||
SQLiteTxn txn(state->db);
|
||||
if (sqlite3_exec(state->db, "alter table ValidPaths add column ultimate integer", 0, 0, 0) != SQLITE_OK)
|
||||
throwSQLiteError(state->db, "upgrading database schema");
|
||||
if (sqlite3_exec(state->db, "alter table ValidPaths add column sigs text", 0, 0, 0) != SQLITE_OK)
|
||||
throwSQLiteError(state->db, "upgrading database schema");
|
||||
txn.commit();
|
||||
}
|
||||
|
||||
|
|
@ -202,14 +203,16 @@ LocalStore::LocalStore()
|
|||
lockFile(globalLock, ltRead, true);
|
||||
}
|
||||
|
||||
else openDB(false);
|
||||
else openDB(*state, false);
|
||||
}
|
||||
|
||||
|
||||
LocalStore::~LocalStore()
|
||||
{
|
||||
auto state(_state.lock());
|
||||
|
||||
try {
|
||||
for (auto & i : runningSubstituters) {
|
||||
for (auto & i : state->runningSubstituters) {
|
||||
if (i.second.disabled) continue;
|
||||
i.second.to.close();
|
||||
i.second.from.close();
|
||||
|
|
@ -222,9 +225,9 @@ LocalStore::~LocalStore()
|
|||
}
|
||||
|
||||
try {
|
||||
if (fdTempRoots != -1) {
|
||||
fdTempRoots.close();
|
||||
unlink(fnTempRoots.c_str());
|
||||
if (state->fdTempRoots != -1) {
|
||||
state->fdTempRoots.close();
|
||||
unlink(state->fnTempRoots.c_str());
|
||||
}
|
||||
} catch (...) {
|
||||
ignoreException();
|
||||
|
|
@ -250,13 +253,14 @@ bool LocalStore::haveWriteAccess()
|
|||
}
|
||||
|
||||
|
||||
void LocalStore::openDB(bool create)
|
||||
void LocalStore::openDB(State & state, bool create)
|
||||
{
|
||||
if (!haveWriteAccess())
|
||||
throw SysError(format("Nix database directory ‘%1%’ is not writable") % settings.nixDBPath);
|
||||
|
||||
/* Open the Nix database. */
|
||||
string dbPath = settings.nixDBPath + "/db.sqlite";
|
||||
auto & db(state.db);
|
||||
if (sqlite3_open_v2(dbPath.c_str(), &db.db,
|
||||
SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
|
||||
throw Error(format("cannot open Nix database ‘%1%’") % dbPath);
|
||||
|
|
@ -309,41 +313,41 @@ void LocalStore::openDB(bool create)
|
|||
}
|
||||
|
||||
/* Prepare SQL statements. */
|
||||
stmtRegisterValidPath.create(db,
|
||||
state.stmtRegisterValidPath.create(db,
|
||||
"insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs) values (?, ?, ?, ?, ?, ?, ?);");
|
||||
stmtUpdatePathInfo.create(db,
|
||||
state.stmtUpdatePathInfo.create(db,
|
||||
"update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ? where path = ?;");
|
||||
stmtAddReference.create(db,
|
||||
state.stmtAddReference.create(db,
|
||||
"insert or replace into Refs (referrer, reference) values (?, ?);");
|
||||
stmtQueryPathInfo.create(db,
|
||||
state.stmtQueryPathInfo.create(db,
|
||||
"select id, hash, registrationTime, deriver, narSize, ultimate, sigs from ValidPaths where path = ?;");
|
||||
stmtQueryReferences.create(db,
|
||||
state.stmtQueryReferences.create(db,
|
||||
"select path from Refs join ValidPaths on reference = id where referrer = ?;");
|
||||
stmtQueryReferrers.create(db,
|
||||
state.stmtQueryReferrers.create(db,
|
||||
"select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);");
|
||||
stmtInvalidatePath.create(db,
|
||||
state.stmtInvalidatePath.create(db,
|
||||
"delete from ValidPaths where path = ?;");
|
||||
stmtRegisterFailedPath.create(db,
|
||||
state.stmtRegisterFailedPath.create(db,
|
||||
"insert or ignore into FailedPaths (path, time) values (?, ?);");
|
||||
stmtHasPathFailed.create(db,
|
||||
state.stmtHasPathFailed.create(db,
|
||||
"select time from FailedPaths where path = ?;");
|
||||
stmtQueryFailedPaths.create(db,
|
||||
state.stmtQueryFailedPaths.create(db,
|
||||
"select path from FailedPaths;");
|
||||
// If the path is a derivation, then clear its outputs.
|
||||
stmtClearFailedPath.create(db,
|
||||
state.stmtClearFailedPath.create(db,
|
||||
"delete from FailedPaths where ?1 = '*' or path = ?1 "
|
||||
"or path in (select d.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where v.path = ?1);");
|
||||
stmtAddDerivationOutput.create(db,
|
||||
state.stmtAddDerivationOutput.create(db,
|
||||
"insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);");
|
||||
stmtQueryValidDerivers.create(db,
|
||||
state.stmtQueryValidDerivers.create(db,
|
||||
"select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;");
|
||||
stmtQueryDerivationOutputs.create(db,
|
||||
state.stmtQueryDerivationOutputs.create(db,
|
||||
"select id, path from DerivationOutputs where drv = ?;");
|
||||
// Use "path >= ?" with limit 1 rather than "path like '?%'" to
|
||||
// ensure efficient lookup.
|
||||
stmtQueryPathFromHashPart.create(db,
|
||||
state.stmtQueryPathFromHashPart.create(db,
|
||||
"select path from ValidPaths where path >= ? limit 1;");
|
||||
stmtQueryValidPaths.create(db, "select path from ValidPaths");
|
||||
state.stmtQueryValidPaths.create(db, "select path from ValidPaths");
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -538,9 +542,10 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation &
|
|||
}
|
||||
|
||||
|
||||
uint64_t LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs)
|
||||
uint64_t LocalStore::addValidPath(State & state,
|
||||
const ValidPathInfo & info, bool checkOutputs)
|
||||
{
|
||||
stmtRegisterValidPath.use()
|
||||
state.stmtRegisterValidPath.use()
|
||||
(info.path)
|
||||
("sha256:" + printHash(info.narHash))
|
||||
(info.registrationTime == 0 ? time(0) : info.registrationTime)
|
||||
|
|
@ -549,7 +554,7 @@ uint64_t LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs)
|
|||
(info.ultimate ? 1 : 0, info.ultimate)
|
||||
(concatStringsSep(" ", info.sigs), !info.sigs.empty())
|
||||
.exec();
|
||||
uint64_t id = sqlite3_last_insert_rowid(db);
|
||||
uint64_t id = sqlite3_last_insert_rowid(state.db);
|
||||
|
||||
/* If this is a derivation, then store the derivation outputs in
|
||||
the database. This is useful for the garbage collector: it can
|
||||
|
|
@ -566,7 +571,7 @@ uint64_t LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs)
|
|||
if (checkOutputs) checkDerivationOutputs(info.path, drv);
|
||||
|
||||
for (auto & i : drv.outputs) {
|
||||
stmtAddDerivationOutput.use()
|
||||
state.stmtAddDerivationOutput.use()
|
||||
(id)
|
||||
(i.first)
|
||||
(i.second.path)
|
||||
|
|
@ -578,16 +583,11 @@ uint64_t LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs)
|
|||
}
|
||||
|
||||
|
||||
void LocalStore::addReference(uint64_t referrer, uint64_t reference)
|
||||
{
|
||||
stmtAddReference.use()(referrer)(reference).exec();
|
||||
}
|
||||
|
||||
|
||||
void LocalStore::registerFailedPath(const Path & path)
|
||||
{
|
||||
retrySQLite<void>([&]() {
|
||||
stmtRegisterFailedPath.use()(path)(time(0)).step();
|
||||
auto state(_state.lock());
|
||||
state->stmtRegisterFailedPath.use()(path)(time(0)).step();
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -595,7 +595,8 @@ void LocalStore::registerFailedPath(const Path & path)
|
|||
bool LocalStore::hasPathFailed(const Path & path)
|
||||
{
|
||||
return retrySQLite<bool>([&]() {
|
||||
return stmtHasPathFailed.use()(path).next();
|
||||
auto state(_state.lock());
|
||||
return state->stmtHasPathFailed.use()(path).next();
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -603,7 +604,9 @@ bool LocalStore::hasPathFailed(const Path & path)
|
|||
PathSet LocalStore::queryFailedPaths()
|
||||
{
|
||||
return retrySQLite<PathSet>([&]() {
|
||||
auto useQueryFailedPaths(stmtQueryFailedPaths.use());
|
||||
auto state(_state.lock());
|
||||
|
||||
auto useQueryFailedPaths(state->stmtQueryFailedPaths.use());
|
||||
|
||||
PathSet res;
|
||||
while (useQueryFailedPaths.next())
|
||||
|
|
@ -617,10 +620,12 @@ PathSet LocalStore::queryFailedPaths()
|
|||
void LocalStore::clearFailedPaths(const PathSet & paths)
|
||||
{
|
||||
retrySQLite<void>([&]() {
|
||||
SQLiteTxn txn(db);
|
||||
auto state(_state.lock());
|
||||
|
||||
SQLiteTxn txn(state->db);
|
||||
|
||||
for (auto & path : paths)
|
||||
stmtClearFailedPath.use()(path).exec();
|
||||
state->stmtClearFailedPath.use()(path).exec();
|
||||
|
||||
txn.commit();
|
||||
});
|
||||
|
|
@ -649,9 +654,10 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path)
|
|||
assertStorePath(path);
|
||||
|
||||
return retrySQLite<ValidPathInfo>([&]() {
|
||||
auto state(_state.lock());
|
||||
|
||||
/* Get the path info. */
|
||||
auto useQueryPathInfo(stmtQueryPathInfo.use()(path));
|
||||
auto useQueryPathInfo(state->stmtQueryPathInfo.use()(path));
|
||||
|
||||
if (!useQueryPathInfo.next())
|
||||
throw Error(format("path ‘%1%’ is not valid") % path);
|
||||
|
|
@ -662,19 +668,19 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path)
|
|||
|
||||
info.registrationTime = useQueryPathInfo.getInt(2);
|
||||
|
||||
auto s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3);
|
||||
auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3);
|
||||
if (s) info.deriver = s;
|
||||
|
||||
/* Note that narSize = NULL yields 0. */
|
||||
info.narSize = useQueryPathInfo.getInt(4);
|
||||
|
||||
info.ultimate = sqlite3_column_int(stmtQueryPathInfo, 5) == 1;
|
||||
info.ultimate = useQueryPathInfo.getInt(5) == 1;
|
||||
|
||||
s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 6);
|
||||
s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 6);
|
||||
if (s) info.sigs = tokenizeString<StringSet>(s, " ");
|
||||
|
||||
/* Get the references. */
|
||||
auto useQueryReferences(stmtQueryReferences.use()(info.id));
|
||||
auto useQueryReferences(state->stmtQueryReferences.use()(info.id));
|
||||
|
||||
while (useQueryReferences.next())
|
||||
info.references.insert(useQueryReferences.getStr(0));
|
||||
|
|
@ -685,9 +691,9 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path)
|
|||
|
||||
|
||||
/* Update path info in the database. */
|
||||
void LocalStore::updatePathInfo(const ValidPathInfo & info)
|
||||
void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
|
||||
{
|
||||
stmtUpdatePathInfo.use()
|
||||
state.stmtUpdatePathInfo.use()
|
||||
(info.narSize, info.narSize != 0)
|
||||
("sha256:" + printHash(info.narHash))
|
||||
(info.ultimate ? 1 : 0, info.ultimate)
|
||||
|
|
@ -697,44 +703,44 @@ void LocalStore::updatePathInfo(const ValidPathInfo & info)
|
|||
}
|
||||
|
||||
|
||||
uint64_t LocalStore::queryValidPathId(const Path & path)
|
||||
uint64_t LocalStore::queryValidPathId(State & state, const Path & path)
|
||||
{
|
||||
auto use(stmtQueryPathInfo.use()(path));
|
||||
auto use(state.stmtQueryPathInfo.use()(path));
|
||||
if (!use.next())
|
||||
throw Error(format("path ‘%1%’ is not valid") % path);
|
||||
return use.getInt(0);
|
||||
}
|
||||
|
||||
|
||||
bool LocalStore::isValidPath_(const Path & path)
|
||||
bool LocalStore::isValidPath(State & state, const Path & path)
|
||||
{
|
||||
return stmtQueryPathInfo.use()(path).next();
|
||||
return state.stmtQueryPathInfo.use()(path).next();
|
||||
}
|
||||
|
||||
|
||||
bool LocalStore::isValidPath(const Path & path)
|
||||
{
|
||||
return retrySQLite<bool>([&]() {
|
||||
return isValidPath_(path);
|
||||
auto state(_state.lock());
|
||||
return isValidPath(*state, path);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
PathSet LocalStore::queryValidPaths(const PathSet & paths)
|
||||
{
|
||||
return retrySQLite<PathSet>([&]() {
|
||||
PathSet res;
|
||||
for (auto & i : paths)
|
||||
if (isValidPath_(i)) res.insert(i);
|
||||
return res;
|
||||
});
|
||||
PathSet res;
|
||||
for (auto & i : paths)
|
||||
if (isValidPath(i)) res.insert(i);
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
PathSet LocalStore::queryAllValidPaths()
|
||||
{
|
||||
return retrySQLite<PathSet>([&]() {
|
||||
auto use(stmtQueryValidPaths.use());
|
||||
auto state(_state.lock());
|
||||
auto use(state->stmtQueryValidPaths.use());
|
||||
PathSet res;
|
||||
while (use.next()) res.insert(use.getStr(0));
|
||||
return res;
|
||||
|
|
@ -742,9 +748,9 @@ PathSet LocalStore::queryAllValidPaths()
|
|||
}
|
||||
|
||||
|
||||
void LocalStore::queryReferrers_(const Path & path, PathSet & referrers)
|
||||
void LocalStore::queryReferrers(State & state, const Path & path, PathSet & referrers)
|
||||
{
|
||||
auto useQueryReferrers(stmtQueryReferrers.use()(path));
|
||||
auto useQueryReferrers(state.stmtQueryReferrers.use()(path));
|
||||
|
||||
while (useQueryReferrers.next())
|
||||
referrers.insert(useQueryReferrers.getStr(0));
|
||||
|
|
@ -755,7 +761,8 @@ void LocalStore::queryReferrers(const Path & path, PathSet & referrers)
|
|||
{
|
||||
assertStorePath(path);
|
||||
return retrySQLite<void>([&]() {
|
||||
queryReferrers_(path, referrers);
|
||||
auto state(_state.lock());
|
||||
queryReferrers(*state, path, referrers);
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -771,7 +778,9 @@ PathSet LocalStore::queryValidDerivers(const Path & path)
|
|||
assertStorePath(path);
|
||||
|
||||
return retrySQLite<PathSet>([&]() {
|
||||
auto useQueryValidDerivers(stmtQueryValidDerivers.use()(path));
|
||||
auto state(_state.lock());
|
||||
|
||||
auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(path));
|
||||
|
||||
PathSet derivers;
|
||||
while (useQueryValidDerivers.next())
|
||||
|
|
@ -785,7 +794,10 @@ PathSet LocalStore::queryValidDerivers(const Path & path)
|
|||
PathSet LocalStore::queryDerivationOutputs(const Path & path)
|
||||
{
|
||||
return retrySQLite<PathSet>([&]() {
|
||||
auto useQueryDerivationOutputs(stmtQueryDerivationOutputs.use()(queryValidPathId(path)));
|
||||
auto state(_state.lock());
|
||||
|
||||
auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()
|
||||
(queryValidPathId(*state, path)));
|
||||
|
||||
PathSet outputs;
|
||||
while (useQueryDerivationOutputs.next())
|
||||
|
|
@ -799,7 +811,10 @@ PathSet LocalStore::queryDerivationOutputs(const Path & path)
|
|||
StringSet LocalStore::queryDerivationOutputNames(const Path & path)
|
||||
{
|
||||
return retrySQLite<StringSet>([&]() {
|
||||
auto useQueryDerivationOutputs(stmtQueryDerivationOutputs.use()(queryValidPathId(path)));
|
||||
auto state(_state.lock());
|
||||
|
||||
auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()
|
||||
(queryValidPathId(*state, path)));
|
||||
|
||||
StringSet outputNames;
|
||||
while (useQueryDerivationOutputs.next())
|
||||
|
|
@ -817,11 +832,13 @@ Path LocalStore::queryPathFromHashPart(const string & hashPart)
|
|||
Path prefix = settings.nixStore + "/" + hashPart;
|
||||
|
||||
return retrySQLite<Path>([&]() {
|
||||
auto useQueryPathFromHashPart(stmtQueryPathFromHashPart.use()(prefix));
|
||||
auto state(_state.lock());
|
||||
|
||||
auto useQueryPathFromHashPart(state->stmtQueryPathFromHashPart.use()(prefix));
|
||||
|
||||
if (!useQueryPathFromHashPart.next()) return "";
|
||||
|
||||
const char * s = (const char *) sqlite3_column_text(stmtQueryPathFromHashPart, 0);
|
||||
const char * s = (const char *) sqlite3_column_text(state->stmtQueryPathFromHashPart, 0);
|
||||
return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : "";
|
||||
});
|
||||
}
|
||||
|
|
@ -829,13 +846,13 @@ Path LocalStore::queryPathFromHashPart(const string & hashPart)
|
|||
|
||||
void LocalStore::setSubstituterEnv()
|
||||
{
|
||||
if (didSetSubstituterEnv) return;
|
||||
static std::atomic_flag done;
|
||||
|
||||
if (done.test_and_set()) return;
|
||||
|
||||
/* Pass configuration options (including those overridden with
|
||||
--option) to substituters. */
|
||||
setenv("_NIX_OPTIONS", settings.pack().c_str(), 1);
|
||||
|
||||
didSetSubstituterEnv = true;
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -957,10 +974,12 @@ template<class T> T LocalStore::getIntLineFromSubstituter(RunningSubstituter & r
|
|||
|
||||
PathSet LocalStore::querySubstitutablePaths(const PathSet & paths)
|
||||
{
|
||||
auto state(_state.lock());
|
||||
|
||||
PathSet res;
|
||||
for (auto & i : settings.substituters) {
|
||||
if (res.size() == paths.size()) break;
|
||||
RunningSubstituter & run(runningSubstituters[i]);
|
||||
RunningSubstituter & run(state->runningSubstituters[i]);
|
||||
startSubstituter(i, run);
|
||||
if (run.disabled) continue;
|
||||
string s = "have ";
|
||||
|
|
@ -977,6 +996,7 @@ PathSet LocalStore::querySubstitutablePaths(const PathSet & paths)
|
|||
res.insert(path);
|
||||
}
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
|
|
@ -984,7 +1004,9 @@ PathSet LocalStore::querySubstitutablePaths(const PathSet & paths)
|
|||
void LocalStore::querySubstitutablePathInfos(const Path & substituter,
|
||||
PathSet & paths, SubstitutablePathInfos & infos)
|
||||
{
|
||||
RunningSubstituter & run(runningSubstituters[substituter]);
|
||||
auto state(_state.lock());
|
||||
|
||||
RunningSubstituter & run(state->runningSubstituters[substituter]);
|
||||
startSubstituter(substituter, run);
|
||||
if (run.disabled) return;
|
||||
|
||||
|
|
@ -1048,22 +1070,24 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
|
|||
if (settings.syncBeforeRegistering) sync();
|
||||
|
||||
return retrySQLite<void>([&]() {
|
||||
SQLiteTxn txn(db);
|
||||
auto state(_state.lock());
|
||||
|
||||
SQLiteTxn txn(state->db);
|
||||
PathSet paths;
|
||||
|
||||
for (auto & i : infos) {
|
||||
assert(i.narHash.type == htSHA256);
|
||||
if (isValidPath_(i.path))
|
||||
updatePathInfo(i);
|
||||
if (isValidPath(*state, i.path))
|
||||
updatePathInfo(*state, i);
|
||||
else
|
||||
addValidPath(i, false);
|
||||
addValidPath(*state, i, false);
|
||||
paths.insert(i.path);
|
||||
}
|
||||
|
||||
for (auto & i : infos) {
|
||||
auto referrer = queryValidPathId(i.path);
|
||||
auto referrer = queryValidPathId(*state, i.path);
|
||||
for (auto & j : i.references)
|
||||
addReference(referrer, queryValidPathId(j));
|
||||
state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)).exec();
|
||||
}
|
||||
|
||||
/* Check that the derivation outputs are correct. We can't do
|
||||
|
|
@ -1090,13 +1114,11 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
|
|||
|
||||
/* Invalidate a path. The caller is responsible for checking that
|
||||
there are no referrers. */
|
||||
void LocalStore::invalidatePath(const Path & path)
|
||||
void LocalStore::invalidatePath(State & state, const Path & path)
|
||||
{
|
||||
debug(format("invalidating path ‘%1%’") % path);
|
||||
|
||||
drvHashes.erase(path);
|
||||
|
||||
stmtInvalidatePath.use()(path).exec();
|
||||
state.stmtInvalidatePath.use()(path).exec();
|
||||
|
||||
/* Note that the foreign key constraints on the Refs table take
|
||||
care of deleting the references entries for `path'. */
|
||||
|
|
@ -1465,15 +1487,17 @@ void LocalStore::invalidatePathChecked(const Path & path)
|
|||
assertStorePath(path);
|
||||
|
||||
retrySQLite<void>([&]() {
|
||||
SQLiteTxn txn(db);
|
||||
auto state(_state.lock());
|
||||
|
||||
if (isValidPath_(path)) {
|
||||
PathSet referrers; queryReferrers_(path, referrers);
|
||||
SQLiteTxn txn(state->db);
|
||||
|
||||
if (isValidPath(*state, path)) {
|
||||
PathSet referrers; queryReferrers(*state, path, referrers);
|
||||
referrers.erase(path); /* ignore self-references */
|
||||
if (!referrers.empty())
|
||||
throw PathInUse(format("cannot delete path ‘%1%’ because it is in use by %2%")
|
||||
% path % showPaths(referrers));
|
||||
invalidatePath(path);
|
||||
invalidatePath(*state, path);
|
||||
}
|
||||
|
||||
txn.commit();
|
||||
|
|
@ -1542,7 +1566,10 @@ bool LocalStore::verifyStore(bool checkContents, bool repair)
|
|||
update = true;
|
||||
}
|
||||
|
||||
if (update) updatePathInfo(info);
|
||||
if (update) {
|
||||
auto state(_state.lock());
|
||||
updatePathInfo(*state, info);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -1572,7 +1599,8 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store,
|
|||
|
||||
if (!isStorePath(path)) {
|
||||
printMsg(lvlError, format("path ‘%1%’ is not in the Nix store") % path);
|
||||
invalidatePath(path);
|
||||
auto state(_state.lock());
|
||||
invalidatePath(*state, path);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -1590,7 +1618,8 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store,
|
|||
|
||||
if (canInvalidate) {
|
||||
printMsg(lvlError, format("path ‘%1%’ disappeared, removing from database...") % path);
|
||||
invalidatePath(path);
|
||||
auto state(_state.lock());
|
||||
invalidatePath(*state, path);
|
||||
} else {
|
||||
printMsg(lvlError, format("path ‘%1%’ disappeared, but it still has valid referrers!") % path);
|
||||
if (repair)
|
||||
|
|
@ -1610,32 +1639,6 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store,
|
|||
}
|
||||
|
||||
|
||||
bool LocalStore::pathContentsGood(const Path & path)
|
||||
{
|
||||
std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path);
|
||||
if (i != pathContentsGoodCache.end()) return i->second;
|
||||
printMsg(lvlInfo, format("checking path ‘%1%’...") % path);
|
||||
ValidPathInfo info = queryPathInfo(path);
|
||||
bool res;
|
||||
if (!pathExists(path))
|
||||
res = false;
|
||||
else {
|
||||
HashResult current = hashPath(info.narHash.type, path);
|
||||
Hash nullHash(htSHA256);
|
||||
res = info.narHash == nullHash || info.narHash == current.first;
|
||||
}
|
||||
pathContentsGoodCache[path] = res;
|
||||
if (!res) printMsg(lvlError, format("path ‘%1%’ is corrupted or missing!") % path);
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
void LocalStore::markContentsGood(const Path & path)
|
||||
{
|
||||
pathContentsGoodCache[path] = true;
|
||||
}
|
||||
|
||||
|
||||
#if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL)
|
||||
|
||||
static void makeMutable(const Path & path)
|
||||
|
|
@ -1690,21 +1693,25 @@ void LocalStore::upgradeStore7()
|
|||
|
||||
void LocalStore::vacuumDB()
|
||||
{
|
||||
if (sqlite3_exec(db, "vacuum;", 0, 0, 0) != SQLITE_OK)
|
||||
throwSQLiteError(db, "vacuuming SQLite database");
|
||||
auto state(_state.lock());
|
||||
|
||||
if (sqlite3_exec(state->db, "vacuum;", 0, 0, 0) != SQLITE_OK)
|
||||
throwSQLiteError(state->db, "vacuuming SQLite database");
|
||||
}
|
||||
|
||||
|
||||
void LocalStore::addSignatures(const Path & storePath, const StringSet & sigs)
|
||||
{
|
||||
retrySQLite<void>([&]() {
|
||||
SQLiteTxn txn(db);
|
||||
auto state(_state.lock());
|
||||
|
||||
SQLiteTxn txn(state->db);
|
||||
|
||||
auto info = queryPathInfo(storePath);
|
||||
|
||||
info.sigs.insert(sigs.begin(), sigs.end());
|
||||
|
||||
updatePathInfo(info);
|
||||
updatePathInfo(*state, info);
|
||||
|
||||
txn.commit();
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,13 +1,15 @@
|
|||
#pragma once
|
||||
|
||||
#include "sqlite.hh"
|
||||
#include <string>
|
||||
#include <unordered_set>
|
||||
|
||||
#include "pathlocks.hh"
|
||||
#include "store-api.hh"
|
||||
#include "sync.hh"
|
||||
#include "util.hh"
|
||||
|
||||
#include <string>
|
||||
#include <unordered_set>
|
||||
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
|
@ -52,12 +54,47 @@ struct RunningSubstituter
|
|||
class LocalStore : public LocalFSStore
|
||||
{
|
||||
private:
|
||||
typedef std::map<Path, RunningSubstituter> RunningSubstituters;
|
||||
RunningSubstituters runningSubstituters;
|
||||
|
||||
Path linksDir;
|
||||
/* Lock file used for upgrading. */
|
||||
AutoCloseFD globalLock;
|
||||
|
||||
Path reservedPath;
|
||||
struct State
|
||||
{
|
||||
/* The SQLite database object. */
|
||||
SQLite db;
|
||||
|
||||
/* Some precompiled SQLite statements. */
|
||||
SQLiteStmt stmtRegisterValidPath;
|
||||
SQLiteStmt stmtUpdatePathInfo;
|
||||
SQLiteStmt stmtAddReference;
|
||||
SQLiteStmt stmtQueryPathInfo;
|
||||
SQLiteStmt stmtQueryReferences;
|
||||
SQLiteStmt stmtQueryReferrers;
|
||||
SQLiteStmt stmtInvalidatePath;
|
||||
SQLiteStmt stmtRegisterFailedPath;
|
||||
SQLiteStmt stmtHasPathFailed;
|
||||
SQLiteStmt stmtQueryFailedPaths;
|
||||
SQLiteStmt stmtClearFailedPath;
|
||||
SQLiteStmt stmtAddDerivationOutput;
|
||||
SQLiteStmt stmtQueryValidDerivers;
|
||||
SQLiteStmt stmtQueryDerivationOutputs;
|
||||
SQLiteStmt stmtQueryPathFromHashPart;
|
||||
SQLiteStmt stmtQueryValidPaths;
|
||||
|
||||
/* The file to which we write our temporary roots. */
|
||||
Path fnTempRoots;
|
||||
AutoCloseFD fdTempRoots;
|
||||
|
||||
typedef std::map<Path, RunningSubstituter> RunningSubstituters;
|
||||
RunningSubstituters runningSubstituters;
|
||||
|
||||
};
|
||||
|
||||
Sync<State, std::recursive_mutex> _state;
|
||||
|
||||
const Path linksDir;
|
||||
const Path reservedPath;
|
||||
const Path schemaPath;
|
||||
|
||||
public:
|
||||
|
||||
|
|
@ -174,76 +211,25 @@ public:
|
|||
a substituter (if available). */
|
||||
void repairPath(const Path & path);
|
||||
|
||||
/* Check whether the given valid path exists and has the right
|
||||
contents. */
|
||||
bool pathContentsGood(const Path & path);
|
||||
|
||||
void markContentsGood(const Path & path);
|
||||
|
||||
void setSubstituterEnv();
|
||||
|
||||
void addSignatures(const Path & storePath, const StringSet & sigs) override;
|
||||
|
||||
private:
|
||||
|
||||
Path schemaPath;
|
||||
|
||||
/* Lock file used for upgrading. */
|
||||
AutoCloseFD globalLock;
|
||||
|
||||
/* The SQLite database object. */
|
||||
SQLite db;
|
||||
|
||||
/* Some precompiled SQLite statements. */
|
||||
SQLiteStmt stmtRegisterValidPath;
|
||||
SQLiteStmt stmtUpdatePathInfo;
|
||||
SQLiteStmt stmtAddReference;
|
||||
SQLiteStmt stmtQueryPathInfo;
|
||||
SQLiteStmt stmtQueryReferences;
|
||||
SQLiteStmt stmtQueryReferrers;
|
||||
SQLiteStmt stmtInvalidatePath;
|
||||
SQLiteStmt stmtRegisterFailedPath;
|
||||
SQLiteStmt stmtHasPathFailed;
|
||||
SQLiteStmt stmtQueryFailedPaths;
|
||||
SQLiteStmt stmtClearFailedPath;
|
||||
SQLiteStmt stmtAddDerivationOutput;
|
||||
SQLiteStmt stmtQueryValidDerivers;
|
||||
SQLiteStmt stmtQueryDerivationOutputs;
|
||||
SQLiteStmt stmtQueryPathFromHashPart;
|
||||
SQLiteStmt stmtQueryValidPaths;
|
||||
|
||||
/* Cache for pathContentsGood(). */
|
||||
std::map<Path, bool> pathContentsGoodCache;
|
||||
|
||||
bool didSetSubstituterEnv;
|
||||
|
||||
/* The file to which we write our temporary roots. */
|
||||
Path fnTempRoots;
|
||||
AutoCloseFD fdTempRoots;
|
||||
|
||||
int getSchema();
|
||||
|
||||
public:
|
||||
|
||||
static bool haveWriteAccess();
|
||||
|
||||
private:
|
||||
|
||||
void openDB(bool create);
|
||||
int getSchema();
|
||||
|
||||
void openDB(State & state, bool create);
|
||||
|
||||
void makeStoreWritable();
|
||||
|
||||
uint64_t queryValidPathId(const Path & path);
|
||||
uint64_t queryValidPathId(State & state, const Path & path);
|
||||
|
||||
uint64_t addValidPath(const ValidPathInfo & info, bool checkOutputs = true);
|
||||
uint64_t addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs = true);
|
||||
|
||||
void addReference(uint64_t referrer, uint64_t reference);
|
||||
|
||||
void appendReferrer(const Path & from, const Path & to, bool lock);
|
||||
|
||||
void rewriteReferrers(const Path & path, bool purge, PathSet referrers);
|
||||
|
||||
void invalidatePath(const Path & path);
|
||||
void invalidatePath(State & state, const Path & path);
|
||||
|
||||
/* Delete a path from the Nix store. */
|
||||
void invalidatePathChecked(const Path & path);
|
||||
|
|
@ -251,7 +237,7 @@ private:
|
|||
void verifyPath(const Path & path, const PathSet & store,
|
||||
PathSet & done, PathSet & validPaths, bool repair, bool & errors);
|
||||
|
||||
void updatePathInfo(const ValidPathInfo & info);
|
||||
void updatePathInfo(State & state, const ValidPathInfo & info);
|
||||
|
||||
void upgradeStore6();
|
||||
void upgradeStore7();
|
||||
|
|
@ -299,8 +285,8 @@ private:
|
|||
void optimisePath_(OptimiseStats & stats, const Path & path, InodeHash & inodeHash);
|
||||
|
||||
// Internal versions that are not wrapped in retry_sqlite.
|
||||
bool isValidPath_(const Path & path);
|
||||
void queryReferrers_(const Path & path, PathSet & referrers);
|
||||
bool isValidPath(State & state, const Path & path);
|
||||
void queryReferrers(State & state, const Path & path, PathSet & referrers);
|
||||
|
||||
/* Add signatures to a ValidPathInfo using the secret keys
|
||||
specified by the ‘secret-key-files’ option. */
|
||||
|
|
|
|||
|
|
@ -22,11 +22,11 @@ namespace nix {
|
|||
scope.
|
||||
*/
|
||||
|
||||
template<class T>
|
||||
template<class T, class M = std::mutex>
|
||||
class Sync
|
||||
{
|
||||
private:
|
||||
std::mutex mutex;
|
||||
M mutex;
|
||||
T data;
|
||||
|
||||
public:
|
||||
|
|
@ -38,7 +38,7 @@ public:
|
|||
{
|
||||
private:
|
||||
Sync * s;
|
||||
std::unique_lock<std::mutex> lk;
|
||||
std::unique_lock<M> lk;
|
||||
friend Sync;
|
||||
Lock(Sync * s) : s(s), lk(s->mutex) { }
|
||||
public:
|
||||
|
|
|
|||
Loading…
Reference in a new issue