diff --git a/src/libexpr/flake/eval-cache.cc b/src/libexpr/flake/eval-cache.cc
new file mode 100644
index 000000000..b32d502f7
--- /dev/null
+++ b/src/libexpr/flake/eval-cache.cc
@@ -0,0 +1,116 @@
+#include "eval-cache.hh"
+#include "sqlite.hh"
+#include "eval.hh"
+
+#include <set>
+
+namespace nix::flake {
+
+static const char * schema = R"sql(
+
+create table if not exists Fingerprints (
+ fingerprint blob primary key not null,
+ timestamp integer not null
+);
+
+create table if not exists Attributes (
+ fingerprint blob not null,
+ attrPath text not null,
+ type integer,
+ value text,
+ primary key (fingerprint, attrPath),
+ foreign key (fingerprint) references Fingerprints(fingerprint) on delete cascade
+);
+)sql";
+
+struct EvalCache::State
+{
+ SQLite db;
+ SQLiteStmt insertFingerprint;
+ SQLiteStmt insertAttribute;
+ SQLiteStmt queryAttribute;
+ std::set<Fingerprint> fingerprints;
+};
+
+EvalCache::EvalCache()
+ : _state(std::make_unique<Sync<State>>())
+{
+ auto state(_state->lock());
+
+ Path dbPath = getCacheDir() + "/nix/eval-cache-v1.sqlite";
+ createDirs(dirOf(dbPath));
+
+ state->db = SQLite(dbPath);
+ state->db.isCache();
+ state->db.exec(schema);
+
+ state->insertFingerprint.create(state->db,
+ "insert or ignore into Fingerprints(fingerprint, timestamp) values (?, ?)");
+
+ state->insertAttribute.create(state->db,
+ "insert or replace into Attributes(fingerprint, attrPath, type, value) values (?, ?, ?, ?)");
+
+ state->queryAttribute.create(state->db,
+ "select type, value from Attributes where fingerprint = ? and attrPath = ?");
+}
+
+enum ValueType {
+ Derivation = 1,
+};
+
+void EvalCache::addDerivation(
+ const Fingerprint & fingerprint,
+ const std::string & attrPath,
+ const Derivation & drv)
+{
+ if (!evalSettings.pureEval) return;
+
+ auto state(_state->lock());
+
+ if (state->fingerprints.insert(fingerprint).second)
+ // FIXME: update timestamp
+ state->insertFingerprint.use()
+ (fingerprint.hash, fingerprint.hashSize)
+ (time(0)).exec();
+
+ state->insertAttribute.use()
+ (fingerprint.hash, fingerprint.hashSize)
+ (attrPath)
+ (ValueType::Derivation)
+ (drv.drvPath + " " + drv.outPath + " " + drv.outputName).exec();
+}
+
+std::optional<EvalCache::Derivation> EvalCache::getDerivation(
+ const Fingerprint & fingerprint,
+ const std::string & attrPath)
+{
+ if (!evalSettings.pureEval) return {};
+
+ auto state(_state->lock());
+
+ auto queryAttribute(state->queryAttribute.use()
+ (fingerprint.hash, fingerprint.hashSize)
+ (attrPath));
+ if (!queryAttribute.next()) return {};
+
+ // FIXME: handle negative results
+
+ auto type = (ValueType) queryAttribute.getInt(0);
+ auto s = queryAttribute.getStr(1);
+
+ if (type != ValueType::Derivation) return {};
+
+ auto ss = tokenizeString<std::vector<std::string>>(s, " ");
+
+ debug("evaluation cache hit for '%s'", attrPath);
+
+ return Derivation { ss[0], ss[1], ss[2] };
+}
+
+EvalCache & EvalCache::singleton()
+{
+ static std::unique_ptr<EvalCache> evalCache(new EvalCache());
+ return *evalCache;
+}
+
+}
diff --git a/src/libexpr/flake/eval-cache.hh b/src/libexpr/flake/eval-cache.hh
new file mode 100644
index 000000000..03aea142e
--- /dev/null
+++ b/src/libexpr/flake/eval-cache.hh
@@ -0,0 +1,39 @@
+#pragma once
+
+#include "sync.hh"
+#include "flake.hh"
+
+namespace nix { struct SQLite; struct SQLiteStmt; }
+
+namespace nix::flake {
+
+class EvalCache
+{
+ struct State;
+
+ std::unique_ptr<Sync<State>> _state;
+
+ EvalCache();
+
+public:
+
+ struct Derivation
+ {
+ Path drvPath;
+ Path outPath;
+ std::string outputName;
+ };
+
+ void addDerivation(
+ const Fingerprint & fingerprint,
+ const std::string & attrPath,
+ const Derivation & drv);
+
+ std::optional<Derivation> getDerivation(
+ const Fingerprint & fingerprint,
+ const std::string & attrPath);
+
+ static EvalCache & singleton();
+};
+
+}
diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index bb0543541..0018a0d07 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -601,4 +601,13 @@ const Registries EvalState::getFlakeRegistries()
return registries;
}
+Fingerprint ResolvedFlake::getFingerprint() const
+{
+ // FIXME: as an optimization, if the flake contains a lockfile and
+ // we haven't changed it, then it's sufficient to use
+ // flake.sourceInfo.storePath for the fingerprint.
+ return hashString(htSHA256,
+ fmt("%s;%s", flake.sourceInfo.storePath, lockFile));
+}
+
}
diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh
index b8d0da252..81b6541f0 100644
--- a/src/libexpr/flake/flake.hh
+++ b/src/libexpr/flake/flake.hh
@@ -83,12 +83,18 @@ struct NonFlake
Flake getFlake(EvalState &, const FlakeRef &, bool impureIsAllowed);
+/* Fingerprint of a locked flake; used as a cache key. */
+typedef Hash Fingerprint;
+
struct ResolvedFlake
{
Flake flake;
LockFile lockFile;
+
ResolvedFlake(Flake && flake, LockFile && lockFile)
: flake(flake), lockFile(lockFile) {}
+
+ Fingerprint getFingerprint() const;
};
ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, HandleLockFile);
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 485fdd691..f39c73b23 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -294,9 +294,7 @@ void LocalStore::openDB(State & state, bool create)
/* Open the Nix database. */
string dbPath = dbDir + "/db.sqlite";
auto & db(state.db);
- if (sqlite3_open_v2(dbPath.c_str(), &db.db,
- SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
- throw Error(format("cannot open Nix database '%1%'") % dbPath);
+ state.db = SQLite(dbPath, create);
#ifdef __CYGWIN__
/* The cygwin version of sqlite3 has a patch which calls
@@ -308,11 +306,6 @@ void LocalStore::openDB(State & state, bool create)
SetDllDirectoryW(L"");
#endif
- if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
- throwSQLiteError(db, "setting timeout");
-
- db.exec("pragma foreign_keys = 1");
-
/* !!! check whether sqlite has been built with foreign key
support */
diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index 32ad7f2b2..3f6dbbcf5 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -78,12 +78,7 @@ public:
state->db = SQLite(dbPath);
- if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK)
- throwSQLiteError(state->db, "setting timeout");
-
- // We can always reproduce the cache.
- state->db.exec("pragma synchronous = off");
- state->db.exec("pragma main.journal_mode = truncate");
+ state->db.isCache();
state->db.exec(schema);
diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc
index a061d64f3..eb1daafc5 100644
--- a/src/libstore/sqlite.cc
+++ b/src/libstore/sqlite.cc
@@ -25,11 +25,16 @@ namespace nix {
throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path);
}
-SQLite::SQLite(const Path & path)
+SQLite::SQLite(const Path & path, bool create)
{
if (sqlite3_open_v2(path.c_str(), &db,
- SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK)
+ SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
throw Error(format("cannot open SQLite database '%s'") % path);
+
+ if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
+ throwSQLiteError(db, "setting timeout");
+
+ exec("pragma foreign_keys = 1");
}
SQLite::~SQLite()
@@ -42,6 +47,12 @@ SQLite::~SQLite()
}
}
+void SQLite::isCache()
+{
+ exec("pragma synchronous = off");
+ exec("pragma main.journal_mode = truncate");
+}
+
void SQLite::exec(const std::string & stmt)
{
retrySQLite<void>([&]() {
@@ -94,6 +105,16 @@ SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool
return *this;
}
+SQLiteStmt::Use & SQLiteStmt::Use::operator () (const unsigned char * data, size_t len, bool notNull)
+{
+ if (notNull) {
+ if (sqlite3_bind_blob(stmt, curArg++, data, len, SQLITE_TRANSIENT) != SQLITE_OK)
+ throwSQLiteError(stmt.db, "binding argument");
+ } else
+ bind();
+ return *this;
+}
+
SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull)
{
if (notNull) {
diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh
index 115679b84..78e53fa32 100644
--- a/src/libstore/sqlite.hh
+++ b/src/libstore/sqlite.hh
@@ -15,13 +15,16 @@ struct SQLite
{
sqlite3 * db = 0;
SQLite() { }
- SQLite(const Path & path);
+ SQLite(const Path & path, bool create = true);
SQLite(const SQLite & from) = delete;
SQLite& operator = (const SQLite & from) = delete;
SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; }
~SQLite();
operator sqlite3 * () { return db; }
+ /* Disable synchronous mode, set truncate journal mode. */
+ void isCache();
+
void exec(const std::string & stmt);
};
@@ -52,6 +55,7 @@ struct SQLiteStmt
/* Bind the next parameter. */
Use & operator () (const std::string & value, bool notNull = true);
+ Use & operator () (const unsigned char * data, size_t len, bool notNull = true);
Use & operator () (int64_t value, bool notNull = true);
Use & bind(); // null
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index a85295a09..86e601bc4 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -8,6 +8,7 @@
#include "store-api.hh"
#include "shared.hh"
#include "flake/flake.hh"
+#include "flake/eval-cache.hh"
#include <regex>
#include <queue>
@@ -110,7 +111,7 @@ struct InstallableValue : Installable
InstallableValue(SourceExprCommand & cmd) : cmd(cmd) { }
- Buildables toBuildables() override
+ virtual std::vector<flake::EvalCache::Derivation> toDerivations()
{
auto state = cmd.getEvalState();
@@ -118,22 +119,36 @@ struct InstallableValue : Installable
Bindings & autoArgs = *cmd.getAutoArgs(*state);
- DrvInfos drvs;
- getDerivations(*state, *v, "", autoArgs, drvs, false);
+ DrvInfos drvInfos;
+ getDerivations(*state, *v, "", autoArgs, drvInfos, false);
+ std::vector<flake::EvalCache::Derivation> res;
+ for (auto & drvInfo : drvInfos) {
+ res.push_back({
+ drvInfo.queryDrvPath(),
+ drvInfo.queryOutPath(),
+ drvInfo.queryOutputName()
+ });
+ }
+
+ return res;
+ }
+
+ Buildables toBuildables() override
+ {
Buildables res;
PathSet drvPaths;
- for (auto & drv : drvs) {
- Buildable b{drv.queryDrvPath()};
+ for (auto & drv : toDerivations()) {
+ Buildable b{drv.drvPath};
drvPaths.insert(b.drvPath);
- auto outputName = drv.queryOutputName();
+ auto outputName = drv.outputName;
if (outputName == "")
throw Error("derivation '%s' lacks an 'outputName' attribute", b.drvPath);
- b.outputs.emplace(outputName, drv.queryOutPath());
+ b.outputs.emplace(outputName, drv.outPath);
res.push_back(std::move(b));
}
@@ -254,12 +269,30 @@ struct InstallableFlake : InstallableValue
std::string what() override { return flakeRef.to_string() + ":" + *attrPaths.begin(); }
- Value * toValue(EvalState & state) override
+ std::vector<std::string> getActualAttrPaths()
+ {
+ std::vector<std::string> res;
+
+ if (searchPackages) {
+ // As a convenience, look for the attribute in
+ // 'outputs.packages'.
+ res.push_back("packages." + *attrPaths.begin());
+
+ // As a temporary hack until Nixpkgs is properly converted
+ // to provide a clean 'packages' set, look in 'legacyPackages'.
+ res.push_back("legacyPackages." + *attrPaths.begin());
+ }
+
+ for (auto & s : attrPaths)
+ res.push_back(s);
+
+ return res;
+ }
+
+ Value * getFlakeOutputs(EvalState & state, const flake::ResolvedFlake & resFlake)
{
auto vFlake = state.allocValue();
- auto resFlake = resolveFlake(state, flakeRef, cmd.getLockFileMode());
-
callFlake(state, resFlake, *vFlake);
makeFlakeClosureGCRoot(*state.store, flakeRef, resFlake);
@@ -268,34 +301,67 @@ struct InstallableFlake : InstallableValue
state.forceValue(*vOutputs);
- auto emptyArgs = state.allocBindings(0);
+ return vOutputs;
+ }
- if (searchPackages) {
- // As a convenience, look for the attribute in
- // 'outputs.packages'.
- if (auto aPackages = *vOutputs->attrs->get(state.symbols.create("packages"))) {
- try {
- auto * v = findAlongAttrPath(state, *attrPaths.begin(), *emptyArgs, *aPackages->value);
- state.forceValue(*v);
- return v;
- } catch (AttrPathNotFound & e) {
- }
+ std::vector<flake::EvalCache::Derivation> toDerivations() override
+ {
+ auto state = cmd.getEvalState();
+
+ auto resFlake = resolveFlake(*state, flakeRef, cmd.getLockFileMode());
+
+ Value * vOutputs = nullptr;
+
+ auto emptyArgs = state->allocBindings(0);
+
+ auto & evalCache = flake::EvalCache::singleton();
+
+ auto fingerprint = resFlake.getFingerprint();
+
+ for (auto & attrPath : getActualAttrPaths()) {
+ auto drv = evalCache.getDerivation(fingerprint, attrPath);
+ if (drv) {
+ if (state->store->isValidPath(drv->drvPath))
+ return {*drv};
}
- // As a temporary hack until Nixpkgs is properly converted
- // to provide a clean 'packages' set, look in 'legacyPackages'.
- if (auto aPackages = *vOutputs->attrs->get(state.symbols.create("legacyPackages"))) {
- try {
- auto * v = findAlongAttrPath(state, *attrPaths.begin(), *emptyArgs, *aPackages->value);
- state.forceValue(*v);
- return v;
- } catch (AttrPathNotFound & e) {
- }
+ if (!vOutputs)
+ vOutputs = getFlakeOutputs(*state, resFlake);
+
+ try {
+ auto * v = findAlongAttrPath(*state, attrPath, *emptyArgs, *vOutputs);
+ state->forceValue(*v);
+
+ auto drvInfo = getDerivation(*state, *v, false);
+ if (!drvInfo)
+ throw Error("flake output attribute '%s' is not a derivation", attrPath);
+
+ auto drv = flake::EvalCache::Derivation{
+ drvInfo->queryDrvPath(),
+ drvInfo->queryOutPath(),
+ drvInfo->queryOutputName()
+ };
+
+ evalCache.addDerivation(fingerprint, attrPath, drv);
+
+ return {drv};
+ } catch (AttrPathNotFound & e) {
}
}
- // Otherwise, look for it in 'outputs'.
- for (auto & attrPath : attrPaths) {
+ throw Error("flake '%s' does not provide attribute %s",
+ flakeRef, concatStringsSep(", ", quoteStrings(attrPaths)));
+ }
+
+ Value * toValue(EvalState & state) override
+ {
+ auto resFlake = resolveFlake(state, flakeRef, cmd.getLockFileMode());
+
+ auto vOutputs = getFlakeOutputs(state, resFlake);
+
+ auto emptyArgs = state.allocBindings(0);
+
+ for (auto & attrPath : getActualAttrPaths()) {
try {
auto * v = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs);
state.forceValue(*v);