nix-copy-closure: Implement in C++.
Tests fail currently because the database is not given proper hashes in the VM
This commit is contained in:
parent
8af062f372
commit
bfa41eb671
11 changed files with 100 additions and 115 deletions
2
.gitignore
vendored
2
.gitignore
vendored
|
@ -81,6 +81,8 @@ Makefile.config
|
|||
# /src/nix-build/
|
||||
/src/nix-build/nix-build
|
||||
|
||||
/src/nix-copy-closure/nix-copy-closure
|
||||
|
||||
/src/build-remote/build-remote
|
||||
|
||||
# /tests/
|
||||
|
|
1
Makefile
1
Makefile
|
@ -11,6 +11,7 @@ makefiles = \
|
|||
src/nix-env/local.mk \
|
||||
src/nix-daemon/local.mk \
|
||||
src/nix-collect-garbage/local.mk \
|
||||
src/nix-copy-closure/local.mk \
|
||||
src/nix-prefetch-url/local.mk \
|
||||
src/buildenv/local.mk \
|
||||
src/resolve-system-dependencies/local.mk \
|
||||
|
|
|
@ -1,8 +1,3 @@
|
|||
nix_bin_scripts := \
|
||||
$(d)/nix-copy-closure \
|
||||
|
||||
bin-scripts += $(nix_bin_scripts)
|
||||
|
||||
nix_noinst_scripts := \
|
||||
$(d)/build-remote.pl \
|
||||
$(d)/nix-http-export.cgi \
|
||||
|
@ -16,4 +11,4 @@ profiledir = $(sysconfdir)/profile.d
|
|||
$(eval $(call install-file-as, $(d)/nix-profile.sh, $(profiledir)/nix.sh, 0644))
|
||||
$(eval $(call install-program-in, $(d)/build-remote.pl, $(libexecdir)/nix))
|
||||
|
||||
clean-files += $(nix_bin_scripts) $(nix_noinst_scripts)
|
||||
clean-files += $(nix_noinst_scripts)
|
||||
|
|
|
@ -1,103 +0,0 @@
|
|||
#! @perl@ -w @perlFlags@
|
||||
|
||||
use utf8;
|
||||
use strict;
|
||||
use Nix::SSH;
|
||||
use Nix::Config;
|
||||
use Nix::Store;
|
||||
use Nix::CopyClosure;
|
||||
use List::Util qw(sum);
|
||||
|
||||
binmode STDERR, ":encoding(utf8)";
|
||||
|
||||
if (scalar @ARGV < 1) {
|
||||
print STDERR <<EOF
|
||||
Usage: nix-copy-closure [--from | --to] HOSTNAME [--gzip] [--bzip2] [--xz] PATHS...
|
||||
EOF
|
||||
;
|
||||
exit 1;
|
||||
}
|
||||
|
||||
|
||||
# Get the target host.
|
||||
my $sshHost;
|
||||
my $toMode = 1;
|
||||
my $includeOutputs = 0;
|
||||
my $dryRun = 0;
|
||||
my $useSubstitutes = 0;
|
||||
my $verbosity = 1;
|
||||
|
||||
|
||||
# !!! Copied from nix-pack-closure, should put this in a module.
|
||||
my @storePaths = ();
|
||||
|
||||
while (@ARGV) {
|
||||
my $arg = shift @ARGV;
|
||||
|
||||
if ($arg eq "--help") {
|
||||
exec "man nix-copy-closure" or die;
|
||||
}
|
||||
elsif ($arg eq "--gzip" || $arg eq "--bzip2" || $arg eq "--xz") {
|
||||
warn "$0: ‘$arg’ is not implemented\n" if $arg ne "--gzip";
|
||||
push @globalSshOpts, "-C";
|
||||
}
|
||||
elsif ($arg eq "--from") {
|
||||
$toMode = 0;
|
||||
}
|
||||
elsif ($arg eq "--to") {
|
||||
$toMode = 1;
|
||||
}
|
||||
elsif ($arg eq "--include-outputs") {
|
||||
$includeOutputs = 1;
|
||||
}
|
||||
elsif ($arg eq "--show-progress") {
|
||||
warn "$0: ‘$arg’ is not implemented\n";
|
||||
}
|
||||
elsif ($arg eq "--dry-run") {
|
||||
$dryRun = 1;
|
||||
}
|
||||
elsif ($arg eq "--use-substitutes" || $arg eq "-s") {
|
||||
$useSubstitutes = 1;
|
||||
}
|
||||
elsif ($arg eq "-v") {
|
||||
$verbosity++;
|
||||
setVerbosity($verbosity);
|
||||
}
|
||||
elsif (!defined $sshHost) {
|
||||
$sshHost = $arg;
|
||||
}
|
||||
else {
|
||||
push @storePaths, $arg;
|
||||
}
|
||||
}
|
||||
|
||||
die "$0: you did not specify a host name\n" unless defined $sshHost;
|
||||
|
||||
|
||||
if ($toMode) { # Copy TO the remote machine.
|
||||
Nix::CopyClosure::copyTo(
|
||||
$sshHost, [ @storePaths ],
|
||||
$includeOutputs, $dryRun, $useSubstitutes);
|
||||
}
|
||||
|
||||
else { # Copy FROM the remote machine.
|
||||
|
||||
my ($from, $to) = connectToRemoteNix($sshHost, []);
|
||||
|
||||
# Query the closure of the given store paths on the remote
|
||||
# machine. Paths are assumed to be store paths; there is no
|
||||
# resolution (following of symlinks).
|
||||
syswrite($to, pack("L<x4L<x4", 7, $includeOutputs ? 1 : 0)) or die;
|
||||
writeStrings(\@storePaths, $to);
|
||||
my @missing = grep { !isValidPath($_) } readStrings($from);
|
||||
|
||||
# Export the store paths on the remote machine and import them locally.
|
||||
if (scalar @missing > 0) {
|
||||
print STDERR "copying ", scalar @missing, " missing paths from ‘$sshHost’...\n";
|
||||
writeInt(5, $to); # == cmdExportPaths
|
||||
writeInt(0, $to); # obsolete
|
||||
writeStrings(\@missing, $to);
|
||||
importPaths(fileno($from), 1);
|
||||
}
|
||||
|
||||
}
|
|
@ -9,7 +9,6 @@
|
|||
#include "shared.hh"
|
||||
#include "pathlocks.hh"
|
||||
#include "globals.hh"
|
||||
#include "serve-protocol.hh"
|
||||
#include "serialise.hh"
|
||||
#include "store-api.hh"
|
||||
#include "derivations.hh"
|
||||
|
|
|
@ -8,4 +8,4 @@ build-remote_LIBS = libmain libutil libformat libstore
|
|||
|
||||
build-remote_SOURCES := $(d)/build-remote.cc
|
||||
|
||||
build-remote_CXXFLAGS = -DSYSCONFDIR="\"$(sysconfdir)\"" -Isrc/nix-store
|
||||
build-remote_CXXFLAGS = -DSYSCONFDIR="\"$(sysconfdir)\""
|
||||
|
|
|
@ -39,6 +39,8 @@ private:
|
|||
string uri;
|
||||
|
||||
Path key;
|
||||
|
||||
bool compress;
|
||||
};
|
||||
|
||||
SSHStore::SSHStore(string uri, const Params & params, size_t maxConnections)
|
||||
|
@ -48,6 +50,7 @@ SSHStore::SSHStore(string uri, const Params & params, size_t maxConnections)
|
|||
, socketPath((Path) tmpDir + "/ssh.sock")
|
||||
, uri(std::move(uri))
|
||||
, key(get(params, "ssh-key", ""))
|
||||
, compress(get(params, "compress", "") == "true")
|
||||
{
|
||||
/* open a connection and perform the handshake to verify all is well */
|
||||
connections->get();
|
||||
|
@ -90,11 +93,12 @@ ref<FSAccessor> SSHStore::getFSAccessor()
|
|||
ref<RemoteStore::Connection> SSHStore::openConnection()
|
||||
{
|
||||
if ((pid_t) sshMaster == -1) {
|
||||
auto flags = compress ? "-NMCS" : "-NMS";
|
||||
sshMaster = startProcess([&]() {
|
||||
if (key.empty())
|
||||
execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), uri.c_str(), NULL);
|
||||
execlp("ssh", "ssh", flags, socketPath.c_str(), uri.c_str(), NULL);
|
||||
else
|
||||
execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), "-i", key.c_str(), uri.c_str(), NULL);
|
||||
execlp("ssh", "ssh", flags, socketPath.c_str(), "-i", key.c_str(), uri.c_str(), NULL);
|
||||
throw SysError("starting ssh master");
|
||||
});
|
||||
}
|
||||
|
|
|
@ -4,6 +4,7 @@
|
|||
#include "util.hh"
|
||||
#include "nar-info-disk-cache.hh"
|
||||
#include "thread-pool.hh"
|
||||
#include "derivations.hh"
|
||||
|
||||
#include <future>
|
||||
|
||||
|
@ -699,8 +700,27 @@ std::list<ref<Store>> getDefaultSubstituters()
|
|||
}
|
||||
|
||||
|
||||
void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths)
|
||||
void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths, bool substitute)
|
||||
{
|
||||
if (substitute) {
|
||||
/* Filter out .drv files (we don't want to build anything). */
|
||||
PathSet paths2;
|
||||
for (auto & path : storePaths)
|
||||
if (!isDerivation(path)) paths2.insert(path);
|
||||
unsigned long long downloadSize, narSize;
|
||||
PathSet willBuild, willSubstitute, unknown;
|
||||
to->queryMissing(PathSet(paths2.begin(), paths2.end()),
|
||||
willBuild, willSubstitute, unknown, downloadSize, narSize);
|
||||
/* FIXME: should use ensurePath(), but it only
|
||||
does one path at a time. */
|
||||
if (!willSubstitute.empty())
|
||||
try {
|
||||
to->buildPaths(willSubstitute);
|
||||
} catch (Error & e) {
|
||||
printMsg(lvlError, format("warning: %1%") % e.msg());
|
||||
}
|
||||
}
|
||||
|
||||
std::string copiedLabel = "copied";
|
||||
|
||||
logger->setExpected(copiedLabel, storePaths.size());
|
||||
|
|
|
@ -625,7 +625,7 @@ void removeTempRoots();
|
|||
ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE"));
|
||||
|
||||
|
||||
void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths);
|
||||
void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths, bool substitute = false);
|
||||
|
||||
enum StoreType {
|
||||
tDaemon,
|
||||
|
|
7
src/nix-copy-closure/local.mk
Normal file
7
src/nix-copy-closure/local.mk
Normal file
|
@ -0,0 +1,7 @@
|
|||
programs += nix-copy-closure
|
||||
|
||||
nix-copy-closure_DIR := $(d)
|
||||
|
||||
nix-copy-closure_LIBS = libmain libutil libformat libstore
|
||||
|
||||
nix-copy-closure_SOURCES := $(d)/nix-copy-closure.cc
|
60
src/nix-copy-closure/nix-copy-closure.cc
Executable file
60
src/nix-copy-closure/nix-copy-closure.cc
Executable file
|
@ -0,0 +1,60 @@
|
|||
#include "shared.hh"
|
||||
#include "store-api.hh"
|
||||
|
||||
using namespace nix;
|
||||
|
||||
int main(int argc, char ** argv)
|
||||
{
|
||||
return handleExceptions(argv[0], [&]() {
|
||||
initNix();
|
||||
auto gzip = false;
|
||||
auto toMode = true;
|
||||
auto includeOutputs = false;
|
||||
auto dryRun = false;
|
||||
auto useSubstitutes = false;
|
||||
auto sshHost = string{};
|
||||
auto storePaths = PathSet{};
|
||||
parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
|
||||
if (*arg == "--help")
|
||||
showManPage("nix-copy-closure");
|
||||
else if (*arg == "--version")
|
||||
printVersion("nix-copy-closure");
|
||||
else if (*arg == "--gzip" || *arg == "--bzip2" || *arg == "--xz") {
|
||||
if (*arg != "--gzip")
|
||||
printMsg(lvlError, format("Warning: ‘%1%’ is not implemented, falling back to gzip") % *arg);
|
||||
gzip = true;
|
||||
} else if (*arg == "--from")
|
||||
toMode = false;
|
||||
else if (*arg == "--to")
|
||||
toMode = true;
|
||||
else if (*arg == "--include-outputs")
|
||||
includeOutputs = true;
|
||||
else if (*arg == "--show-progress")
|
||||
printMsg(lvlError, "Warning: ‘--show-progress’ is not implemented");
|
||||
else if (*arg == "--dry-run")
|
||||
dryRun = true;
|
||||
else if (*arg == "--use-substitutes" || *arg == "-s")
|
||||
useSubstitutes = true;
|
||||
else if (sshHost.empty())
|
||||
sshHost = *arg;
|
||||
else
|
||||
storePaths.insert(*arg);
|
||||
return true;
|
||||
});
|
||||
if (sshHost.empty())
|
||||
throw UsageError("no host name specified");
|
||||
|
||||
auto remoteUri = "ssh://" + sshHost + (gzip ? "?compress=true" : "");
|
||||
auto to = toMode ? openStore(remoteUri) : openStore();
|
||||
auto from = toMode ? openStore() : openStore(remoteUri);
|
||||
if (includeOutputs) {
|
||||
auto newPaths = PathSet{};
|
||||
for (const auto & p : storePaths) {
|
||||
auto outputs = from->queryDerivationOutputs(p);
|
||||
newPaths.insert(outputs.begin(), outputs.end());
|
||||
}
|
||||
storePaths.insert(newPaths.begin(), newPaths.end());
|
||||
}
|
||||
copyPaths(from, to, Paths(storePaths.begin(), storePaths.end()), useSubstitutes);
|
||||
});
|
||||
}
|
Loading…
Reference in a new issue