cafkafk/lix
cafkafk/lix
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Merge "libstore: Create platform LocalStore subclasses" into main

Browse source
This commit is contained in:
Artemis Tosini 2024-04-24 15:35:32 +00:00 committed by Gerrit Code Review
commit 7114b0465a
12 changed files with 269 additions and 120 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
src/libstore/gc-store.hh
View file

@ -7,7 +7,14 @@
namespace nix {
/**
* Garbage-collector roots, referring to a store path
*/
typedef std::unordered_map<StorePath, std::unordered_set<std::string>> Roots;
/**
* Possible garbage collector roots, referring to any path
*/
typedef std::unordered_map<Path, std::unordered_set<std::string>> UncheckedRoots;
struct GCOptions

111
src/libstore/gc.cc
View file

@ -321,105 +321,8 @@ Roots LocalStore::findRoots(bool censor)
return roots;
}
typedef std::unordered_map<Path, std::unordered_set<std::string>> UncheckedRoots;
static void readProcLink(const std::string & file, UncheckedRoots & roots)
void LocalStore::findPlatformRoots(UncheckedRoots & unchecked)
{
constexpr auto bufsiz = PATH_MAX;
char buf[bufsiz];
auto res = readlink(file.c_str(), buf, bufsiz);
if (res == -1) {
if (errno == ENOENT || errno == EACCES || errno == ESRCH)
return;
throw SysError("reading symlink");
}
if (res == bufsiz) {
throw Error("overly long symlink starting with '%1%'", std::string_view(buf, bufsiz));
}
if (res > 0 && buf[0] == '/')
roots[std::string(static_cast<char *>(buf), res)]
.emplace(file);
}
static std::string quoteRegexChars(const std::string & raw)
{
static auto specialRegex = std::regex(R"([.^$\\*+?()\[\]{}|])");
return std::regex_replace(raw, specialRegex, R"(\$&)");
}
#if __linux__
static void readFileRoots(const char * path, UncheckedRoots & roots)
{
try {
roots[readFile(path)].emplace(path);
} catch (SysError & e) {
if (e.errNo != ENOENT && e.errNo != EACCES)
throw;
}
}
#endif
void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
{
UncheckedRoots unchecked;
auto procDir = AutoCloseDir{opendir("/proc")};
if (procDir) {
struct dirent * ent;
auto digitsRegex = std::regex(R"(^\d+$)");
auto mapRegex = std::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)");
auto storePathRegex = std::regex(quoteRegexChars(storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)");
while (errno = 0, ent = readdir(procDir.get())) {
checkInterrupt();
if (std::regex_match(ent->d_name, digitsRegex)) {
try {
readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked);
readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked);
auto fdStr = fmt("/proc/%s/fd", ent->d_name);
auto fdDir = AutoCloseDir(opendir(fdStr.c_str()));
if (!fdDir) {
if (errno == ENOENT || errno == EACCES)
continue;
throw SysError("opening %1%", fdStr);
}
struct dirent * fd_ent;
while (errno = 0, fd_ent = readdir(fdDir.get())) {
if (fd_ent->d_name[0] != '.')
readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked);
}
if (errno) {
if (errno == ESRCH)
continue;
throw SysError("iterating /proc/%1%/fd", ent->d_name);
}
fdDir.reset();
auto mapFile = fmt("/proc/%s/maps", ent->d_name);
auto mapLines = tokenizeString<std::vector<std::string>>(readFile(mapFile), "\n");
for (const auto & line : mapLines) {
auto match = std::smatch{};
if (std::regex_match(line, match, mapRegex))
unchecked[match[1]].emplace(mapFile);
}
auto envFile = fmt("/proc/%s/environ", ent->d_name);
auto envString = readFile(envFile);
auto env_end = std::sregex_iterator{};
for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i)
unchecked[i->str()].emplace(envFile);
} catch (SysError & e) {
if (errno == ENOENT || errno == EACCES || errno == ESRCH)
continue;
throw;
}
}
}
if (errno)
throw SysError("iterating /proc");
}
#if !defined(__linux__)
// lsof is really slow on OS X. This actually causes the gc-concurrent.sh test to fail.
// See: https://github.com/NixOS/nix/issues/3011
// Because of this we disable lsof when running the tests.
@ -437,13 +340,13 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
/* lsof not installed, lsof failed */
}
}
#endif
}
#if __linux__
readFileRoots("/proc/sys/kernel/modprobe", unchecked);
readFileRoots("/proc/sys/kernel/fbsplash", unchecked);
readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked);
#endif
void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
{
UncheckedRoots unchecked;
findPlatformRoots(unchecked);
for (auto & [target, links] : unchecked) {
if (!isInStore(target)) continue;

2
src/libstore/local-store.cc
View file

@ -1940,6 +1940,4 @@ std::optional<std::string> LocalStore::getVersion()
return nixVersion;
}
static RegisterStoreImplementation<LocalStore, LocalStoreConfig> regLocalStore;
} // namespace nix

31
src/libstore/local-store.hh
View file

@ -127,6 +127,17 @@ private:
const PublicKeys & getPublicKeys();
protected:
/**
* Initialise the local store, upgrading the schema if
* necessary.
* Protected so that users don't accidentally create a LocalStore
* instead of a platform's subclass.
*/
LocalStore(const Params & params);
LocalStore(std::string scheme, std::string path, const Params & params);
public:
/**
@ -134,18 +145,16 @@ public:
*/
PathSet locksHeld;
/**
* Initialise the local store, upgrading the schema if
* necessary.
*/
LocalStore(const Params & params);
LocalStore(std::string scheme, std::string path, const Params & params);
~LocalStore();
virtual ~LocalStore();
static std::set<std::string> uriSchemes()
{ return {}; }
/**
* Create a LocalStore, possibly a platform-specific subclass
*/
static std::shared_ptr<LocalStore> makeLocalStore(const Params & params);
/**
* Implementations of abstract store API methods.
*/
@ -330,6 +339,12 @@ private:
void findRootsNoTemp(Roots & roots, bool censor);
/**
* Find possible garbage collector roots in a platform-specific manner,
* e.g. by looking in `/proc` or using `lsof`
*/
virtual void findPlatformRoots(UncheckedRoots & unchecked);
void findRuntimeRoots(Roots & roots, bool censor);
std::pair<Path, AutoCloseFD> createTempDirInStore();

5
src/libstore/local.mk
View file

@ -5,6 +5,11 @@ libstore_NAME = libnixstore
libstore_DIR := $(d)
libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc)
ifdef HOST_LINUX
libstore_SOURCES += $(d)/platform/linux.cc
else
libstore_SOURCES += $(d)/platform/fallback.cc
endif
libstore_LIBS = libutil

9
src/libstore/meson.build
View file

@ -66,6 +66,7 @@ libstore_sources = files(
'path-with-outputs.cc',
'path.cc',
'pathlocks.cc',
'platform.cc',
'profiles.cc',
'realisation.cc',
'remote-fs-accessor.cc',
@ -158,6 +159,14 @@ libstore_headers = files(
'worker-protocol.hh',
)
if host_machine.system() == 'linux'
libstore_sources += files('platform/linux.cc')
libstore_headers += files('platform/linux.hh')
else
libstore_sources += files('platform/fallback.cc')
libstore_headers += files('platform/fallback.hh')
endif
# These variables (aside from LSOF) are created pseudo-dynamically, near the beginning of
# the top-level meson.build. Aside from prefix itself, each of these was
# made into an absolute path by joining it with prefix, unless it was already

18
src/libstore/platform.cc Normal file
View file

@ -0,0 +1,18 @@
#include "local-store.hh"
#if __linux__
#include "platform/linux.hh"
#else
#include "platform/fallback.hh"
#endif
namespace nix {
std::shared_ptr<LocalStore> LocalStore::makeLocalStore(const Params & params)
{
#if __linux__
return std::shared_ptr<LocalStore>(new LinuxLocalStore(params));
#else
return std::shared_ptr<LocalStore>(new FallbackLocalStore(params));
#endif
}
}

5
src/libstore/platform/fallback.cc Normal file
View file

@ -0,0 +1,5 @@
#include "platform/fallback.hh"
namespace nix {
static RegisterStoreImplementation<FallbackLocalStore, LocalStoreConfig> regLocalStore;
}

31
src/libstore/platform/fallback.hh Normal file
View file

@ -0,0 +1,31 @@
#pragma once
///@file
#include "local-store.hh"
namespace nix {
/**
* Fallback platform implementation of LocalStore
* Exists so we can make LocalStore constructor protected
*/
class FallbackLocalStore : public LocalStore
{
public:
FallbackLocalStore(const Params & params)
: StoreConfig(params)
, LocalFSStoreConfig(params)
, LocalStoreConfig(params)
, Store(params)
, LocalFSStore(params)
, LocalStore(params)
{
}
FallbackLocalStore(const std::string scheme, std::string path, const Params & params)
: FallbackLocalStore(params)
{
throw UnimplementedError("FallbackLocalStore");
}
};
}

123
src/libstore/platform/linux.cc Normal file
View file

@ -0,0 +1,123 @@
#include "gc-store.hh"
#include "signals.hh"
#include "platform/linux.hh"
#include <regex>
namespace nix {
static RegisterStoreImplementation<LinuxLocalStore, LocalStoreConfig> regLocalStore;
static void readProcLink(const std::string & file, UncheckedRoots & roots)
{
constexpr auto bufsiz = PATH_MAX;
char buf[bufsiz];
auto res = readlink(file.c_str(), buf, bufsiz);
if (res == -1) {
if (errno == ENOENT || errno == EACCES || errno == ESRCH) {
return;
}
throw SysError("reading symlink");
}
if (res == bufsiz) {
throw Error("overly long symlink starting with '%1%'", std::string_view(buf, bufsiz));
}
if (res > 0 && buf[0] == '/') {
roots[std::string(static_cast<char *>(buf), res)].emplace(file);
}
}
static std::string quoteRegexChars(const std::string & raw)
{
static auto specialRegex = std::regex(R"([.^$\\*+?()\[\]{}|])");
return std::regex_replace(raw, specialRegex, R"(\$&)");
}
static void readFileRoots(const char * path, UncheckedRoots & roots)
{
try {
roots[readFile(path)].emplace(path);
} catch (SysError & e) {
if (e.errNo != ENOENT && e.errNo != EACCES) {
throw;
}
}
}
void LinuxLocalStore::findPlatformRoots(UncheckedRoots & unchecked)
{
auto procDir = AutoCloseDir{opendir("/proc")};
if (procDir) {
struct dirent * ent;
auto digitsRegex = std::regex(R"(^\d+$)");
auto mapRegex = std::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)");
auto storePathRegex =
std::regex(quoteRegexChars(storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)");
while (errno = 0, ent = readdir(procDir.get())) {
checkInterrupt();
if (std::regex_match(ent->d_name, digitsRegex)) {
try {
readProcLink(fmt("/proc/%s/exe", ent->d_name), unchecked);
readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked);
auto fdStr = fmt("/proc/%s/fd", ent->d_name);
auto fdDir = AutoCloseDir(opendir(fdStr.c_str()));
if (!fdDir) {
if (errno == ENOENT || errno == EACCES) {
continue;
}
throw SysError("opening %1%", fdStr);
}
struct dirent * fd_ent;
while (errno = 0, fd_ent = readdir(fdDir.get())) {
if (fd_ent->d_name[0] != '.') {
readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked);
}
}
if (errno) {
if (errno == ESRCH) {
continue;
}
throw SysError("iterating /proc/%1%/fd", ent->d_name);
}
fdDir.reset();
auto mapFile = fmt("/proc/%s/maps", ent->d_name);
auto mapLines =
tokenizeString<std::vector<std::string>>(readFile(mapFile), "\n");
for (const auto & line : mapLines) {
auto match = std::smatch{};
if (std::regex_match(line, match, mapRegex)) {
unchecked[match[1]].emplace(mapFile);
}
}
auto envFile = fmt("/proc/%s/environ", ent->d_name);
auto envString = readFile(envFile);
auto env_end = std::sregex_iterator{};
for (auto i =
std::sregex_iterator{
envString.begin(), envString.end(), storePathRegex
};
i != env_end;
++i)
{
unchecked[i->str()].emplace(envFile);
}
} catch (SysError & e) {
if (errno == ENOENT || errno == EACCES || errno == ESRCH) {
continue;
}
throw;
}
}
}
if (errno) {
throw SysError("iterating /proc");
}
}
readFileRoots("/proc/sys/kernel/modprobe", unchecked);
readFileRoots("/proc/sys/kernel/fbsplash", unchecked);
readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked);
}
}

35
src/libstore/platform/linux.hh Normal file
View file

@ -0,0 +1,35 @@
#pragma once
///@file
#include "gc-store.hh"
#include "local-store.hh"
namespace nix {
/**
* Linux-specific implementation of LocalStore
*/
class LinuxLocalStore : public LocalStore
{
public:
LinuxLocalStore(const Params & params)
: StoreConfig(params)
, LocalFSStoreConfig(params)
, LocalStoreConfig(params)
, Store(params)
, LocalFSStore(params)
, LocalStore(params)
{
}
LinuxLocalStore(const std::string scheme, std::string path, const Params & params)
: LinuxLocalStore(params)
{
throw UnimplementedError("LinuxLocalStore");
}
private:
void findPlatformRoots(UncheckedRoots & unchecked) override;
};
}

12
src/libstore/store-api.cc
View file

@ -1426,7 +1426,7 @@ std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Para
if (uri == "" || uri == "auto") {
auto stateDir = getOr(params, "state", settings.nixStateDir);
if (access(stateDir.c_str(), R_OK | W_OK) == 0)
return std::make_shared<LocalStore>(params);
return LocalStore::makeLocalStore(params);
else if (pathExists(settings.nixDaemonSocketFile))
return std::make_shared<UDSRemoteStore>(params);
#if __linux__
@ -1444,26 +1444,26 @@ std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Para
try {
createDirs(chrootStore);
} catch (Error & e) {
return std::make_shared<LocalStore>(params);
return LocalStore::makeLocalStore(params);
}
warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore);
} else
debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore);
Store::Params params2;
params2["root"] = chrootStore;
return std::make_shared<LocalStore>(params2);
return LocalStore::makeLocalStore(params);
}
#endif
else
return std::make_shared<LocalStore>(params);
return LocalStore::makeLocalStore(params);
} else if (uri == "daemon") {
return std::make_shared<UDSRemoteStore>(params);
} else if (uri == "local") {
return std::make_shared<LocalStore>(params);
return LocalStore::makeLocalStore(params);
} else if (isNonUriPath(uri)) {
Store::Params params2 = params;
params2["root"] = absPath(uri);
return std::make_shared<LocalStore>(params2);
return LocalStore::makeLocalStore(params2);
} else {
return nullptr;
}