From 1b34b69b45106e5bfdbdc0201d3ff4dcd36632f0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 12 Jul 2018 18:44:37 +0200 Subject: [PATCH] nix-prefetch-url: Download file in constant memory Before: $ command time nix-prefetch-url https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.17.6.tar.xz 1.19user 1.02system 0:41.96elapsed 5%CPU (0avgtext+0avgdata 182720maxresident)k After: 1.38user 1.05system 0:39.73elapsed 6%CPU (0avgtext+0avgdata 16204maxresident)k Note however that addToStore() can still take a lot of memory (e.g. RemoteStore::addToStore() is constant space, but LocalStore::addToStore() isn't; that's fixed by https://github.com/edolstra/nix/commit/c94b4fc7ee0c7b322a5f3c7ee784063b47a11d98 though). Fixes #1400. --- src/nix-prefetch-url/nix-prefetch-url.cc | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 50b2c2803..a3b025723 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -9,6 +9,10 @@ #include +#include +#include +#include + using namespace nix; @@ -160,14 +164,20 @@ int main(int argc, char * * argv) auto actualUri = resolveMirrorUri(*state, uri); - /* Download the file. */ - DownloadRequest req(actualUri); - req.decompress = false; - auto result = getDownloader()->download(req); - AutoDelete tmpDir(createTempDir(), true); Path tmpFile = (Path) tmpDir + "/tmp"; - writeFile(tmpFile, *result.data); + + /* Download the file. */ + { + AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0600); + if (!fd) throw SysError("creating temporary file '%s'", tmpFile); + + FdSink sink(fd.get()); + + DownloadRequest req(actualUri); + req.decompress = false; + getDownloader()->download(std::move(req), sink); + } /* Optionally unpack the file. */ if (unpack) { @@ -191,7 +201,7 @@ int main(int argc, char * * argv) /* FIXME: inefficient; addToStore() will also hash this. */ - hash = unpack ? hashPath(ht, tmpFile).first : hashString(ht, *result.data); + hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile); if (expectedHash != Hash(ht) && expectedHash != hash) throw Error(format("hash mismatch for '%1%'") % uri);