cafkafk/lix
cafkafk/lix
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Allow content-addressable paths to have references

Browse source 
This adds a command 'nix make-content-addressable' that rewrites the
specified store paths into content-addressable paths. The advantage of
such paths is that 1) they can be imported without signatures; 2) they
can enable deduplication in cases where derivation changes do not
cause output changes (apart from store path hashes).

For example,

  $ nix make-content-addressable -r nixpkgs.cowsay
  rewrote '/nix/store/g1g31ah55xdia1jdqabv1imf6mcw0nb1-glibc-2.25-49' to '/nix/store/48jfj7bg78a8n4f2nhg269rgw1936vj4-glibc-2.25-49'
  ...
  rewrote '/nix/store/qbi6rzpk0bxjw8lw6azn2mc7ynnn455q-cowsay-3.03+dfsg1-16' to '/nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16'

We can then copy the resulting closure to another store without
signatures:

  $ nix copy --trusted-public-keys '' ---to ~/my-nix /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16

In order to support self-references in content-addressable paths,
these paths are hashed "modulo" self-references, meaning that
self-references are zeroed out during hashing. Somewhat annoyingly,
this means that the NAR hash stored in the Nix database is no longer
necessarily equal to the output of "nix hash-path"; for
content-addressable paths, you need to pass the --modulo flag:

  $ nix path-info --json /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16  | jq -r .[].narHash
  sha256:0ri611gdilz2c9rsibqhsipbfs9vwcqvs811a52i2bnkhv7w9mgw

  $ nix hash-path --type sha256 --base32 /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16
  1ggznh07khq0hz6id09pqws3a8q9pn03ya3c03nwck1kwq8rclzs

  $ nix hash-path --type sha256 --base32 /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16 --modulo iq6g2x4q62xp7y7493bibx0qn5w7xz67
  0ri611gdilz2c9rsibqhsipbfs9vwcqvs811a52i2bnkhv7w9mgw
This commit is contained in:
Eelco Dolstra 2018-03-30 00:56:13 +02:00
parent aabf5c86c9
commit 0abb3ad537
16 changed files with 289 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
src/libstore/build.cc
View file

@ -727,23 +727,6 @@ HookInstance::~HookInstance()
//////////////////////////////////////////////////////////////////////
typedef map<std::string, std::string> StringRewrites;
std::string rewriteStrings(std::string s, const StringRewrites & rewrites)
{
for (auto & i : rewrites) {
size_t j = 0;
while ((j = s.find(i.first, j)) != string::npos)
s.replace(j, i.first.size(), i.second);
}
return s;
}
//////////////////////////////////////////////////////////////////////
typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
class SubstitutionGoal;
@ -865,7 +848,7 @@ private:
#endif
/* Hash rewriting. */
StringRewrites inputRewrites, outputRewrites;
StringMap inputRewrites, outputRewrites;
typedef map<Path, Path> RedirectedOutputs;
RedirectedOutputs redirectedOutputs;

21
src/libstore/local-store.cc
View file

@ -5,6 +5,7 @@
#include "worker-protocol.hh"
#include "derivations.hh"
#include "nar-info.hh"
#include "references.hh"
#include <iostream>
#include <algorithm>
@ -1009,17 +1010,21 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
/* While restoring the path from the NAR, compute the hash
of the NAR. */
HashSink hashSink(htSHA256);
std::unique_ptr<AbstractHashSink> hashSink;
if (info.ca == "")
hashSink = std::make_unique<HashSink>(htSHA256);
else
hashSink = std::make_unique<HashModuloSink>(htSHA256, storePathToHash(info.path));
LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t {
size_t n = source.read(data, len);
hashSink(data, n);
(*hashSink)(data, n);
return n;
});
restorePath(realPath, wrapperSource);
auto hashResult = hashSink.finish();
auto hashResult = hashSink->finish();
if (hashResult.first != info.narHash)
throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s",
@ -1241,7 +1246,15 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
/* Check the content hash (optionally - slow). */
printMsg(lvlTalkative, format("checking contents of '%1%'") % i);
HashResult current = hashPath(info->narHash.type, toRealPath(i));
std::unique_ptr<AbstractHashSink> hashSink;
if (info->ca == "")
hashSink = std::make_unique<HashSink>(info->narHash.type);
else
hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(info->path));
dumpPath(toRealPath(i), *hashSink);
auto current = hashSink->finish();
if (info->narHash != nullHash && info->narHash != current.first) {
printError(format("path '%1%' was modified! "

62
src/libstore/references.cc
View file

@ -118,4 +118,66 @@ PathSet scanForReferences(const string & path,
}
RewritingSink::RewritingSink(const std::string & from, const std::string & to, Sink & nextSink)
: from(from), to(to), nextSink(nextSink)
{
assert(from.size() == to.size());
}
void RewritingSink::operator () (const unsigned char * data, size_t len)
{
std::string s(prev);
s.append((const char *) data, len);
size_t j = 0;
while ((j = s.find(from, j)) != string::npos) {
matches.push_back(pos + j);
s.replace(j, from.size(), to);
}
prev = s.size() < from.size() ? s : std::string(s, s.size() - from.size() + 1, from.size() - 1);
auto consumed = s.size() - prev.size();
pos += consumed;
if (consumed) nextSink((unsigned char *) s.data(), consumed);
}
void RewritingSink::flush()
{
if (prev.empty()) return;
pos += prev.size();
nextSink((unsigned char *) prev.data(), prev.size());
prev.clear();
}
HashModuloSink::HashModuloSink(HashType ht, const std::string & modulus)
: hashSink(ht)
, rewritingSink(modulus, std::string(modulus.size(), 0), hashSink)
{
}
void HashModuloSink::operator () (const unsigned char * data, size_t len)
{
rewritingSink(data, len);
}
HashResult HashModuloSink::finish()
{
rewritingSink.flush();
/* Hash the positions of the self-references. This ensures that a
NAR with self-references and a NAR with some of the
self-references already zeroed out do not produce a hash
collision. FIXME: proof. */
for (auto & pos : rewritingSink.matches) {
auto s = fmt("|%d", pos);
hashSink((unsigned char *) s.data(), s.size());
}
auto h = hashSink.finish();
return {h.first, rewritingSink.pos};
}
}

27
src/libstore/references.hh
View file

@ -8,4 +8,31 @@ namespace nix {
PathSet scanForReferences(const Path & path, const PathSet & refs,
HashResult & hash);
struct RewritingSink : Sink
{
std::string from, to, prev;
Sink & nextSink;
uint64_t pos = 0;
std::vector<uint64_t> matches;
RewritingSink(const std::string & from, const std::string & to, Sink & nextSink);
void operator () (const unsigned char * data, size_t len) override;
void flush();
};
struct HashModuloSink : AbstractHashSink
{
HashSink hashSink;
RewritingSink rewritingSink;
HashModuloSink(HashType ht, const std::string & modulus);
void operator () (const unsigned char * data, size_t len) override;
HashResult finish() override;
};
}

38
src/libstore/store-api.cc
View file

@ -205,15 +205,27 @@ Path Store::makeOutputPath(const string & id,
}
Path Store::makeFixedOutputPath(bool recursive,
const Hash & hash, const string & name) const
static std::string makeType(string && type, const PathSet & references)
{
return hash.type == htSHA256 && recursive
? makeStorePath("source", hash, name)
: makeStorePath("output:out", hashString(htSHA256,
for (auto & i : references) {
type += ":";
type += i;
}
return type;
}
Path Store::makeFixedOutputPath(bool recursive,
const Hash & hash, const string & name, const PathSet & references) const
{
if (hash.type == htSHA256 && recursive) {
return makeStorePath(makeType("source", references), hash, name);
} else {
assert(references.empty());
return makeStorePath("output:out", hashString(htSHA256,
"fixed:out:" + (recursive ? (string) "r:" : "") +
hash.to_string(Base16) + ":"),
name);
hash.to_string(Base16) + ":"), name);
}
}
@ -224,12 +236,7 @@ Path Store::makeTextPath(const string & name, const Hash & hash,
/* Stuff the references (if any) into the type. This is a bit
hacky, but we can't put them in `s' since that would be
ambiguous. */
string type = "text";
for (auto & i : references) {
type += ":";
type += i;
}
return makeStorePath(type, hash, name);
return makeStorePath(makeType("text", references), hash, name);
}
@ -785,8 +792,9 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const
else if (hasPrefix(ca, "fixed:")) {
bool recursive = ca.compare(6, 2, "r:") == 0;
Hash hash(std::string(ca, recursive ? 8 : 6));
if (references.empty() &&
store.makeFixedOutputPath(recursive, hash, storePathToName(path)) == path)
auto refs = references;
replaceInSet(refs, path, std::string("self"));
if (store.makeFixedOutputPath(recursive, hash, storePathToName(path), refs) == path)
return true;
else
warn();

3
src/libstore/store-api.hh
View file

@ -303,7 +303,8 @@ public:
const Hash & hash, const string & name) const;
Path makeFixedOutputPath(bool recursive,
const Hash & hash, const string & name) const;
const Hash & hash, const string & name,
const PathSet & references = {}) const;
Path makeTextPath(const string & name, const Hash & hash,
const PathSet & references) const;

20
src/libutil/hash.cc
View file

@ -256,23 +256,9 @@ Hash hashString(HashType ht, const string & s)
Hash hashFile(HashType ht, const Path & path)
{
Ctx ctx;
Hash hash(ht);
start(ht, ctx);
AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
if (!fd) throw SysError(format("opening file '%1%'") % path);
std::vector<unsigned char> buf(8192);
ssize_t n;
while ((n = read(fd.get(), buf.data(), buf.size()))) {
checkInterrupt();
if (n == -1) throw SysError(format("reading file '%1%'") % path);
update(ht, ctx, buf.data(), n);
}
finish(ht, ctx, hash.hash);
return hash;
HashSink sink(ht);
readFile(path, sink);
return sink.finish().first;
}

11
src/libutil/hash.hh
View file

@ -111,7 +111,12 @@ string printHashType(HashType ht);
union Ctx;
class HashSink : public BufferedSink
struct AbstractHashSink : virtual Sink
{
virtual HashResult finish() = 0;
};
class HashSink : public BufferedSink, public AbstractHashSink
{
private:
HashType ht;
@ -122,8 +127,8 @@ public:
HashSink(HashType ht);
HashSink(const HashSink & h);
~HashSink();
void write(const unsigned char * data, size_t len);
HashResult finish();
void write(const unsigned char * data, size_t len) override;
HashResult finish() override;
HashResult currentHash();
};

2
src/libutil/serialise.hh
View file

@ -24,7 +24,7 @@ struct Sink
/* A buffered abstract sink. */
struct BufferedSink : Sink
struct BufferedSink : virtual Sink
{
size_t bufSize, bufPos;
std::unique_ptr<unsigned char[]> buffer;

13
src/libutil/util.cc
View file

@ -1260,6 +1260,19 @@ string replaceStrings(const std::string & s,
}
std::string rewriteStrings(const std::string & _s, const StringMap & rewrites)
{
auto s = _s;
for (auto & i : rewrites) {
if (i.first == i.second) continue;
size_t j = 0;
while ((j = s.find(i.first, j)) != string::npos)
s.replace(j, i.first.size(), i.second);
}
return s;
}
string statusToString(int status)
{
if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {

14
src/libutil/util.hh
View file

@ -362,6 +362,20 @@ string replaceStrings(const std::string & s,
const std::string & from, const std::string & to);
std::string rewriteStrings(const std::string & s, const StringMap & rewrites);
/* If a set contains 'from', remove it and insert 'to'. */
template<typename T>
void replaceInSet(std::set<T> & set, const T & from, const T & to)
{
auto i = set.find(from);
if (i == set.end()) return;
set.erase(i);
set.insert(to);
}
/* Convert the exit status of a child as returned by wait() into an
error string. */
string statusToString(int status);

2
src/nix/command.cc
View file

@ -129,7 +129,7 @@ void StorePathsCommand::run(ref<Store> store)
}
else {
for (auto & p : toStorePaths(store, NoBuild, installables))
for (auto & p : toStorePaths(store, realiseMode, installables))
storePaths.push_back(p);
if (recursive) {

4
src/nix/command.hh
View file

@ -139,6 +139,10 @@ private:
bool recursive = false;
bool all = false;
protected:
RealiseMode realiseMode = NoBuild;
public:
StorePathsCommand(bool recursive = false);

22
src/nix/hash.cc
View file

@ -2,6 +2,8 @@
#include "hash.hh"
#include "legacy.hh"
#include "shared.hh"
#include "references.hh"
#include "archive.hh"
using namespace nix;
@ -13,6 +15,7 @@ struct CmdHash : Command
bool truncate = false;
HashType ht = htSHA256;
std::vector<std::string> paths;
std::experimental::optional<std::string> modulus;
CmdHash(Mode mode) : mode(mode)
{
@ -23,6 +26,11 @@ struct CmdHash : Command
mkFlag()
.longName("type")
.mkHashTypeFlag(&ht);
mkFlag()
.longName("modulo")
.description("compute hash modulo specified string")
.labels({"modulus"})
.dest(&modulus);
expectArgs("paths", &paths);
}
@ -41,7 +49,19 @@ struct CmdHash : Command
void run() override
{
for (auto path : paths) {
Hash h = mode == mFile ? hashFile(ht, path) : hashPath(ht, path).first;
std::unique_ptr<AbstractHashSink> hashSink;
if (modulus)
hashSink = std::make_unique<HashModuloSink>(ht, *modulus);
else
hashSink = std::make_unique<HashSink>(ht);
if (mode == mFile)
readFile(path, *hashSink);
else
dumpPath(path, *hashSink);
Hash h = hashSink->finish().first;
if (truncate && h.hashSize > 20) h = compressHash(h, 20);
std::cout << format("%1%\n") %
h.to_string(base, base == SRI);

82
src/nix/make-content-addressable.cc Normal file
View file

@ -0,0 +1,82 @@
#include "command.hh"
#include "store-api.hh"
#include "references.hh"
using namespace nix;
struct CmdMakeContentAddressable : StorePathsCommand
{
CmdMakeContentAddressable()
{
realiseMode = Build;
}
std::string name() override
{
return "make-content-addressable";
}
std::string description() override
{
return "test";
}
void run(ref<Store> store, Paths storePaths) override
{
auto paths = store->topoSortPaths(PathSet(storePaths.begin(), storePaths.end()));
paths.reverse();
std::map<Path, Path> remappings;
for (auto & path : paths) {
auto oldInfo = store->queryPathInfo(path);
auto oldHashPart = storePathToHash(path);
auto name = storePathToName(path);
StringSink sink;
store->narFromPath(path, sink);
StringMap rewrites;
ValidPathInfo info;
for (auto & ref : oldInfo->references) {
if (ref == path)
info.references.insert("self");
else {
auto replacement = get(remappings, ref, ref);
// FIXME: warn about unremapped paths?
info.references.insert(replacement);
if (replacement != ref)
rewrites[storePathToHash(ref)] = storePathToHash(replacement);
}
}
*sink.s = rewriteStrings(*sink.s, rewrites);
HashModuloSink hashModuloSink(htSHA256, oldHashPart);
hashModuloSink((unsigned char *) sink.s->data(), sink.s->size());
info.narHash = hashModuloSink.finish().first;
info.narSize = sink.s->size();
replaceInSet(info.references, path, std::string("self"));
info.path = store->makeFixedOutputPath(true, info.narHash, name, info.references);
replaceInSet(info.references, std::string("self"), info.path);
info.ca = makeFixedOutputCA(true, info.narHash);
printError("rewrote '%s' to '%s'", path, info.path);
auto source = sinkToSource([&](Sink & nextSink) {
RewritingSink rsink2(oldHashPart, storePathToHash(info.path), nextSink);
rsink2((unsigned char *) sink.s->data(), sink.s->size());
rsink2.flush();
});
store->addToStore(info, *source);
remappings[path] = info.path;
}
}
};
static RegisterCommand r1(make_ref<CmdMakeContentAddressable>());

12
src/nix/verify.cc
View file

@ -3,6 +3,7 @@
#include "store-api.hh"
#include "sync.hh"
#include "thread-pool.hh"
#include "references.hh"
#include <atomic>
@ -88,10 +89,15 @@ struct CmdVerify : StorePathsCommand
if (!noContents) {
HashSink sink(info->narHash.type);
store->narFromPath(info->path, sink);
std::unique_ptr<AbstractHashSink> hashSink;
if (info->ca == "")
hashSink = std::make_unique<HashSink>(info->narHash.type);
else
hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(info->path));
auto hash = sink.finish();
store->narFromPath(info->path, *hashSink);
auto hash = hashSink->finish();
if (hash.first != info->narHash) {
corrupted++;