Add automatic garbage collection
Nix can now automatically run the garbage collector during builds or while adding paths to the store. The option "min-free = <bytes>" specifies that Nix should run the garbage collector whenever free space in the Nix store drops below <bytes>. It will then delete garbage until "max-free" bytes are available. Garbage collection during builds is asynchronous; running builds are not paused and new builds are not blocked. However, there also is a synchronous GC run prior to the first build/substitution. Currently, no old GC roots are deleted (as in "nix-collect-garbage -d").
This commit is contained in:
parent
b932ea58ec
commit
0b606aad46
6 changed files with 127 additions and 1 deletions
|
@ -393,6 +393,10 @@ configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"
|
|||
package repository.</para>
|
||||
</listitem>
|
||||
|
||||
<listitem>
|
||||
<para>Automatic garbage collection.</para>
|
||||
</listitem>
|
||||
|
||||
</itemizedlist>
|
||||
|
||||
<para>This release has contributions from TBD.</para>
|
||||
|
|
|
@ -3957,6 +3957,8 @@ void Worker::run(const Goals & _topGoals)
|
|||
|
||||
checkInterrupt();
|
||||
|
||||
store.autoGC(false);
|
||||
|
||||
/* Call every wake goal (in the ordering established by
|
||||
CompareGoalPtrs). */
|
||||
while (!awake.empty() && !topGoals.empty()) {
|
||||
|
@ -4014,6 +4016,9 @@ void Worker::waitForInput()
|
|||
is a build timeout, then wait for input until the first
|
||||
deadline for any child. */
|
||||
auto nearest = steady_time_point::max(); // nearest deadline
|
||||
if (settings.minFree.get() != 0)
|
||||
// Periodicallty wake up to see if we need to run the garbage collector.
|
||||
nearest = before + std::chrono::seconds(10);
|
||||
for (auto & i : children) {
|
||||
if (!i.respectTimeouts) continue;
|
||||
if (0 != settings.maxSilentTime)
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
#include "derivations.hh"
|
||||
#include "globals.hh"
|
||||
#include "local-store.hh"
|
||||
#include "finally.hh"
|
||||
|
||||
#include <functional>
|
||||
#include <queue>
|
||||
|
@ -9,6 +10,7 @@
|
|||
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/statvfs.h>
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
|
@ -845,4 +847,72 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
|||
}
|
||||
|
||||
|
||||
void LocalStore::autoGC(bool sync)
|
||||
{
|
||||
auto getAvail = [this]() {
|
||||
struct statvfs st;
|
||||
if (statvfs(realStoreDir.c_str(), &st))
|
||||
throw SysError("getting filesystem info about '%s'", realStoreDir);
|
||||
|
||||
return (uint64_t) st.f_bavail * st.f_bsize;
|
||||
};
|
||||
|
||||
std::shared_future<void> future;
|
||||
|
||||
{
|
||||
auto state(_state.lock());
|
||||
|
||||
if (state->gcRunning) {
|
||||
future = state->gcFuture;
|
||||
debug("waiting for auto-GC to finish");
|
||||
goto sync;
|
||||
}
|
||||
|
||||
auto now = std::chrono::steady_clock::now();
|
||||
|
||||
if (now < state->lastGCCheck + std::chrono::seconds(5)) return;
|
||||
|
||||
auto avail = getAvail();
|
||||
|
||||
state->lastGCCheck = now;
|
||||
|
||||
if (avail >= settings.minFree || avail >= settings.maxFree) return;
|
||||
|
||||
if (avail > state->availAfterGC * 0.97) return;
|
||||
|
||||
state->gcRunning = true;
|
||||
|
||||
std::promise<void> promise;
|
||||
future = state->gcFuture = promise.get_future().share();
|
||||
|
||||
std::thread([promise{std::move(promise)}, this, avail, getAvail]() mutable {
|
||||
|
||||
/* Wake up any threads waiting for the auto-GC to finish. */
|
||||
Finally wakeup([&]() {
|
||||
auto state(_state.lock());
|
||||
state->gcRunning = false;
|
||||
state->lastGCCheck = std::chrono::steady_clock::now();
|
||||
promise.set_value();
|
||||
});
|
||||
|
||||
printInfo("running auto-GC to free %d bytes", settings.maxFree - avail);
|
||||
|
||||
GCOptions options;
|
||||
options.maxFreed = settings.maxFree - avail;
|
||||
|
||||
GCResults results;
|
||||
|
||||
collectGarbage(options, results);
|
||||
|
||||
_state.lock()->availAfterGC = getAvail();
|
||||
|
||||
}).detach();
|
||||
}
|
||||
|
||||
sync:
|
||||
// Wait for the future outside of the state lock.
|
||||
if (sync) future.get();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
@ -4,8 +4,9 @@
|
|||
#include "config.hh"
|
||||
|
||||
#include <map>
|
||||
#include <sys/types.h>
|
||||
#include <limits>
|
||||
|
||||
#include <sys/types.h>
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
@ -342,6 +343,13 @@ public:
|
|||
|
||||
Setting<Strings> hashedMirrors{this, {"http://tarballs.nixos.org/"}, "hashed-mirrors",
|
||||
"A list of servers used by builtins.fetchurl to fetch files by hash."};
|
||||
|
||||
Setting<uint64_t> minFree{this, 0, "min-free",
|
||||
"Automatically run the garbage collector when free disk space drops below the specified amount."};
|
||||
|
||||
Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(), "max-free",
|
||||
"Stop deleting garbage when free disk space is above the specified amount."};
|
||||
|
||||
};
|
||||
|
||||
|
||||
|
|
|
@ -244,6 +244,18 @@ LocalStore::LocalStore(const Params & params)
|
|||
|
||||
LocalStore::~LocalStore()
|
||||
{
|
||||
std::shared_future<void> future;
|
||||
|
||||
{
|
||||
auto state(_state.lock());
|
||||
if (state->gcRunning)
|
||||
future = state->gcFuture;
|
||||
}
|
||||
|
||||
if (future.valid()) {
|
||||
printError("waiting for auto-GC to finish on exit...");
|
||||
future.get();
|
||||
}
|
||||
|
||||
try {
|
||||
auto state(_state.lock());
|
||||
|
@ -991,6 +1003,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> &
|
|||
StringSource source(*nar);
|
||||
restorePath(realPath, source);
|
||||
|
||||
autoGC();
|
||||
|
||||
canonicalisePathMetaData(realPath, -1);
|
||||
|
||||
optimisePath(realPath); // FIXME: combine with hashPath()
|
||||
|
@ -1025,6 +1039,8 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
|
|||
|
||||
deletePath(realPath);
|
||||
|
||||
autoGC();
|
||||
|
||||
if (recursive) {
|
||||
StringSource source(dump);
|
||||
restorePath(realPath, source);
|
||||
|
@ -1097,6 +1113,8 @@ Path LocalStore::addTextToStore(const string & name, const string & s,
|
|||
|
||||
deletePath(realPath);
|
||||
|
||||
autoGC();
|
||||
|
||||
writeFile(realPath, s);
|
||||
|
||||
canonicalisePathMetaData(realPath, -1);
|
||||
|
|
|
@ -7,6 +7,8 @@
|
|||
#include "sync.hh"
|
||||
#include "util.hh"
|
||||
|
||||
#include <chrono>
|
||||
#include <future>
|
||||
#include <string>
|
||||
#include <unordered_set>
|
||||
|
||||
|
@ -60,6 +62,21 @@ private:
|
|||
|
||||
/* The file to which we write our temporary roots. */
|
||||
AutoCloseFD fdTempRoots;
|
||||
|
||||
/* The last time we checked whether to do an auto-GC, or an
|
||||
auto-GC finished. */
|
||||
std::chrono::time_point<std::chrono::steady_clock> lastGCCheck;
|
||||
|
||||
/* Whether auto-GC is running. If so, get gcFuture to wait for
|
||||
the GC to finish. */
|
||||
bool gcRunning = false;
|
||||
std::shared_future<void> gcFuture;
|
||||
|
||||
/* How much disk space was available after the previous
|
||||
auto-GC. If the current available disk space is below
|
||||
minFree but not much below availAfterGC, then there is no
|
||||
point in starting a new GC. */
|
||||
uint64_t availAfterGC = std::numeric_limits<uint64_t>::max();
|
||||
};
|
||||
|
||||
Sync<State, std::recursive_mutex> _state;
|
||||
|
@ -196,6 +213,10 @@ public:
|
|||
|
||||
void addSignatures(const Path & storePath, const StringSet & sigs) override;
|
||||
|
||||
/* If free disk space in /nix/store if below minFree, delete
|
||||
garbage until it exceeds maxFree. */
|
||||
void autoGC(bool sync = true);
|
||||
|
||||
private:
|
||||
|
||||
int getSchema();
|
||||
|
|
Loading…
Reference in a new issue