cafkafk/lix
cafkafk/lix
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/master' into markdown

Browse source
This commit is contained in:
Eelco Dolstra 2020-08-31 14:24:26 +02:00
commit 84f5cabbea
20 changed files with 250 additions and 111 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/test.yml vendored
View file

@ -13,5 +13,7 @@ jobs:
with:
fetch-depth: 0
- uses: cachix/install-nix-action@v10
with:
skip_adding_nixpkgs_channel: true
#- run: nix flake check
- run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi)

9
doc/manual/src/command-ref/opt-common.md
View file

@ -52,8 +52,13 @@ Most Nix commands accept the following command-line options:
This is the raw format, as outputted by nix-build.
- internal-json
Outputs the logs in a structured manner. NOTE: the json schema
is not guarantees to be stable between releases.
Outputs the logs in a structured manner.
> **Warning**
>
> While the schema itself is relatively stable, the format of
> the error-messages (namely of the `msg`-field) can change
> between releases.
- bar
Only display a progress bar during the builds.

8
doc/manual/src/hacking.md
View file

@ -14,7 +14,7 @@ To build Nix for the current operating system/architecture use
$ nix-build
```
or if you have a flakes-enabled nix:
or if you have a flake-enabled nix:
```console
$ nix build
@ -52,6 +52,12 @@ To install it in `$(pwd)/inst` and test it:
nix (Nix) 2.4
```
To run a functional test:
```console
make tests/test-name-should-auto-complete.sh.test
```
If you have a flakes-enabled Nix you can replace:
```console

9
flake.nix
View file

@ -442,6 +442,8 @@
stdenv.mkDerivation {
name = "nix";
outputs = [ "out" "dev" "doc" ];
buildInputs = buildDeps ++ propagatedDeps ++ perlDeps;
inherit configureFlags;
@ -452,12 +454,7 @@
shellHook =
''
export prefix=$(pwd)/inst
configureFlags+=" --prefix=$prefix"
PKG_CONFIG_PATH=$prefix/lib/pkgconfig:$PKG_CONFIG_PATH
PATH=$prefix/bin:$PATH
export MANPATH=/home/eelco/Dev/nix/inst/share/man:$MANPATH
unset PYTHONPATH
export MANPATH=$out/share/man:$MANPATH
'';
});

2
mk/tests.mk
View file

@ -8,7 +8,7 @@ define run-install-test
.PHONY: $1.test
$1.test: $1 $(test-deps)
@env TEST_NAME=$(notdir $(basename $1)) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1
@env TEST_NAME=$(notdir $(basename $1)) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1 < /dev/null
endef

2
perl/Makefile
View file

@ -1,6 +1,6 @@
makefiles = local.mk
GLOBAL_CXXFLAGS += -g -Wall
GLOBAL_CXXFLAGS += -g -Wall -std=c++17
-include Makefile.config

15
src/libexpr/eval.cc
View file

@ -1350,13 +1350,24 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
Value * actualArgs = allocValue();
mkAttrs(*actualArgs, fun.lambda.fun->formals->formals.size());
if (fun.lambda.fun->formals->ellipsis) {
// If the formals have an ellipsis (eg the function accepts extra args) pass
// all available automatic arguments (which includes arguments specified on
// the command line via --arg/--argstr)
for (auto& v : args) {
actualArgs->attrs->push_back(v);
}
} else {
// Otherwise, only pass the arguments that the function accepts
for (auto & i : fun.lambda.fun->formals->formals) {
Bindings::iterator j = args.find(i.name);
if (j != args.end())
if (j != args.end()) {
actualArgs->attrs->push_back(*j);
else if (!i.def)
} else if (!i.def) {
throwTypeError("cannot auto-call a function that has an argument without a default value ('%1%')", i.name);
}
}
}
actualArgs->attrs->sort();

2
src/libexpr/primops.cc
View file

@ -102,7 +102,7 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS
if (auto optStorePath = isValidDerivationInStore()) {
auto storePath = *optStorePath;
Derivation drv = readDerivation(*state.store, realPath, Derivation::nameFromPath(storePath));
Derivation drv = state.store->readDerivation(storePath);
Value & w = *state.allocValue();
state.mkAttrs(w, 3 + drv.outputs.size());
Value * v2 = state.allocAttr(w, state.sDrvPath);

30
src/libstore/derivations.cc
View file

@ -185,7 +185,7 @@ static DerivationOutput parseDerivationOutput(const Store & store, std::istrings
}
static Derivation parseDerivation(const Store & store, std::string && s, std::string_view name)
Derivation parseDerivation(const Store & store, std::string && s, std::string_view name)
{
Derivation drv;
drv.name = name;
@ -233,34 +233,6 @@ static Derivation parseDerivation(const Store & store, std::string && s, std::st
}
Derivation readDerivation(const Store & store, const Path & drvPath, std::string_view name)
{
try {
return parseDerivation(store, readFile(drvPath), name);
} catch (FormatError & e) {
throw Error("error parsing derivation '%1%': %2%", drvPath, e.msg());
}
}
Derivation Store::derivationFromPath(const StorePath & drvPath)
{
ensurePath(drvPath);
return readDerivation(drvPath);
}
Derivation Store::readDerivation(const StorePath & drvPath)
{
auto accessor = getFSAccessor();
try {
return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)), Derivation::nameFromPath(drvPath));
} catch (FormatError & e) {
throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg());
}
}
static void printString(string & res, std::string_view s)
{
char buf[s.size() * 2 + 2];

2
src/libstore/derivations.hh
View file

@ -150,7 +150,7 @@ StorePath writeDerivation(Store & store,
const Derivation & drv, RepairFlag repair = NoRepair);
/* Read a derivation from a file. */
Derivation readDerivation(const Store & store, const Path & drvPath, std::string_view name);
Derivation parseDerivation(const Store & store, std::string && s, std::string_view name);
// FIXME: remove
bool isDerivation(const string & fileName);

3
src/libstore/local-store.hh
View file

@ -23,9 +23,6 @@ namespace nix {
const int nixSchemaVersion = 10;
struct Derivation;
struct OptimiseStats
{
unsigned long filesLinked = 0;

11
src/libstore/remote-store.cc
View file

@ -284,9 +284,9 @@ struct ConnectionHandle
RemoteStore::Connection * operator -> () { return &*handle; }
void processStderr(Sink * sink = 0, Source * source = 0)
void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true)
{
auto ex = handle->processStderr(sink, source);
auto ex = handle->processStderr(sink, source, flush);
if (ex) {
daemonException = true;
std::rethrow_exception(ex);
@ -535,6 +535,8 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) {
conn->to.flush();
std::exception_ptr ex;
struct FramedSink : BufferedSink
@ -574,7 +576,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
std::thread stderrThread([&]()
{
try {
conn.processStderr();
conn.processStderr(nullptr, nullptr, false);
} catch (...) {
ex = std::current_exception();
}
@ -884,8 +886,9 @@ static Logger::Fields readFields(Source & from)
}
std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source)
std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source, bool flush)
{
if (flush)
to.flush();
while (true) {

2
src/libstore/remote-store.hh
View file

@ -114,7 +114,7 @@ protected:
virtual ~Connection();
std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0);
std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true);
};
ref<Connection> openConnectionWrapper();

22
src/libstore/store-api.cc
View file

@ -1,11 +1,11 @@
#include "crypto.hh"
#include "fs-accessor.hh"
#include "globals.hh"
#include "store-api.hh"
#include "util.hh"
#include "nar-info-disk-cache.hh"
#include "thread-pool.hh"
#include "json.hh"
#include "derivations.hh"
#include "url.hh"
#include "archive.hh"
@ -983,6 +983,26 @@ Strings ValidPathInfo::shortRefs() const
}
Derivation Store::derivationFromPath(const StorePath & drvPath)
{
ensurePath(drvPath);
return readDerivation(drvPath);
}
Derivation Store::readDerivation(const StorePath & drvPath)
{
auto accessor = getFSAccessor();
try {
return parseDerivation(*this,
accessor->readFile(printStorePath(drvPath)),
Derivation::nameFromPath(drvPath));
} catch (FormatError & e) {
throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg());
}
}
}

27
src/libutil/logging.cc
View file

@ -184,6 +184,33 @@ struct JSONLogger : Logger {
json["action"] = "msg";
json["level"] = ei.level;
json["msg"] = oss.str();
json["raw_msg"] = ei.hint->str();
if (ei.errPos.has_value() && (*ei.errPos)) {
json["line"] = ei.errPos->line;
json["column"] = ei.errPos->column;
json["file"] = ei.errPos->file;
} else {
json["line"] = nullptr;
json["column"] = nullptr;
json["file"] = nullptr;
}
if (loggerSettings.showTrace.get() && !ei.traces.empty()) {
nlohmann::json traces = nlohmann::json::array();
for (auto iter = ei.traces.rbegin(); iter != ei.traces.rend(); ++iter) {
nlohmann::json stackFrame;
stackFrame["raw_msg"] = iter->hint.str();
if (iter->pos.has_value() && (*iter->pos)) {
stackFrame["line"] = iter->pos->line;
stackFrame["column"] = iter->pos->column;
stackFrame["file"] = iter->pos->file;
}
traces.push_back(stackFrame);
}
json["trace"] = traces;
}
write(json);
}

6
src/libutil/serialise.hh
View file

@ -23,7 +23,8 @@ struct Sink
};
/* A buffered abstract sink. */
/* A buffered abstract sink. Warning: a BufferedSink should not be
used from multiple threads concurrently. */
struct BufferedSink : virtual Sink
{
size_t bufSize, bufPos;
@ -66,7 +67,8 @@ struct Source
};
/* A buffered abstract source. */
/* A buffered abstract source. Warning: a BufferedSource should not be
used from multiple threads concurrently. */
struct BufferedSource : Source
{
size_t bufSize, bufPosIn, bufPosOut;

18
src/libutil/tests/logging.cc
View file

@ -34,6 +34,24 @@ namespace nix {
}
}
TEST(logEI, jsonOutput) {
SymbolTable testTable;
auto problem_file = testTable.create("random.nix");
testing::internal::CaptureStderr();
makeJSONLogger(*logger)->logEI({
.name = "error name",
.description = "error without any code lines.",
.hint = hintfmt("this hint has %1% templated %2%!!",
"yellow",
"values"),
.errPos = Pos(foFile, problem_file, 02, 13)
});
auto str = testing::internal::GetCapturedStderr();
ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1mrandom.nix\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n@nix {\"action\":\"msg\",\"column\":13,\"file\":\"random.nix\",\"level\":0,\"line\":2,\"msg\":\"\\u001b[31;1merror:\\u001b[0m\\u001b[34;1m --- error name --- error-unit-test\\u001b[0m\\n\\u001b[34;1mat: \\u001b[33;1m(2:13)\\u001b[34;1m in file: \\u001b[0mrandom.nix\\n\\nerror without any code lines.\\n\\nthis hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\",\"raw_msg\":\"this hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\"}\n");
}
TEST(logEI, appendingHintsToPreviousError) {
MakeError(TestError, Error);

134
src/nix/develop.cc
View file

@ -15,7 +15,7 @@ struct Var
{
bool exported = true;
bool associative = false;
std::string value; // quoted string or array
std::string quoted; // quoted string or array
};
struct BuildEnvironment
@ -75,12 +75,12 @@ BuildEnvironment readEnvironment(const Path & path)
else if (std::regex_search(pos, file.cend(), match, varRegex, std::regex_constants::match_continuous)) {
pos = match[0].second;
res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .value = match[2] }});
res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .quoted = match[2] }});
}
else if (std::regex_search(pos, file.cend(), match, assocArrayRegex, std::regex_constants::match_continuous)) {
pos = match[0].second;
res.env.insert({match[1], Var { .associative = true, .value = match[2] }});
res.env.insert({match[1], Var { .associative = true, .quoted = match[2] }});
}
else if (std::regex_search(pos, file.cend(), match, functionRegex, std::regex_constants::match_continuous)) {
@ -92,6 +92,8 @@ BuildEnvironment readEnvironment(const Path & path)
path, file.substr(pos - file.cbegin(), 60));
}
res.env.erase("__output");
return res;
}
@ -125,27 +127,32 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
/* Rehash and write the derivation. FIXME: would be nice to use
'buildDerivation', but that's privileged. */
drv.name += "-env";
for (auto & output : drv.outputs)
drv.env.erase(output.first);
drv.outputs = {{"out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = StorePath::dummy }}}};
drv.env["out"] = "";
drv.env["_outputs_saved"] = drv.env["outputs"];
drv.env["outputs"] = "out";
for (auto & output : drv.outputs) {
output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } };
drv.env[output.first] = "";
}
drv.inputSrcs.insert(std::move(getEnvShPath));
Hash h = std::get<0>(hashDerivationModulo(*store, drv, true));
auto shellOutPath = store->makeOutputPath("out", h, drv.name);
drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed {
.path = shellOutPath
} });
drv.env["out"] = store->printStorePath(shellOutPath);
auto shellDrvPath2 = writeDerivation(*store, drv);
for (auto & output : drv.outputs) {
auto outPath = store->makeOutputPath(output.first, h, drv.name);
output.second = { .output = DerivationOutputInputAddressed { .path = outPath } };
drv.env[output.first] = store->printStorePath(outPath);
}
auto shellDrvPath = writeDerivation(*store, drv);
/* Build the derivation. */
store->buildPaths({{shellDrvPath2}});
store->buildPaths({{shellDrvPath}});
assert(store->isValidPath(shellOutPath));
for (auto & outPath : drv.outputPaths(*store)) {
assert(store->isValidPath(outPath));
auto outPathS = store->toRealPath(outPath);
if (lstat(outPathS).st_size)
return outPath;
}
return shellOutPath;
throw Error("get-env.sh failed to produce an environment");
}
struct Common : InstallableCommand, MixProfile
@ -171,8 +178,12 @@ struct Common : InstallableCommand, MixProfile
"UID",
};
void makeRcScript(const BuildEnvironment & buildEnvironment, std::ostream & out)
std::string makeRcScript(
const BuildEnvironment & buildEnvironment,
const Path & outputsDir = absPath(".") + "/outputs")
{
std::ostringstream out;
out << "unset shellHook\n";
out << "nix_saved_PATH=\"$PATH\"\n";
@ -180,9 +191,9 @@ struct Common : InstallableCommand, MixProfile
for (auto & i : buildEnvironment.env) {
if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) {
if (i.second.associative)
out << fmt("declare -A %s=(%s)\n", i.first, i.second.value);
out << fmt("declare -A %s=(%s)\n", i.first, i.second.quoted);
else {
out << fmt("%s=%s\n", i.first, i.second.value);
out << fmt("%s=%s\n", i.first, i.second.quoted);
if (i.second.exported)
out << fmt("export %s\n", i.first);
}
@ -193,13 +204,26 @@ struct Common : InstallableCommand, MixProfile
out << buildEnvironment.bashFunctions << "\n";
// FIXME: set outputs
out << "export NIX_BUILD_TOP=\"$(mktemp -d --tmpdir nix-shell.XXXXXX)\"\n";
for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"})
out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i);
out << "eval \"$shellHook\"\n";
/* Substitute occurrences of output paths. */
auto outputs = buildEnvironment.env.find("outputs");
assert(outputs != buildEnvironment.env.end());
// FIXME: properly unquote 'outputs'.
StringMap rewrites;
for (auto & outputName : tokenizeString<std::vector<std::string>>(replaceStrings(outputs->second.quoted, "'", ""))) {
auto from = buildEnvironment.env.find(outputName);
assert(from != buildEnvironment.env.end());
// FIXME: unquote
rewrites.insert({from->second.quoted, outputsDir + "/" + outputName});
}
return rewriteStrings(out.str(), rewrites);
}
Strings getDefaultFlakeAttrPaths() override
@ -240,6 +264,7 @@ struct Common : InstallableCommand, MixProfile
struct CmdDevelop : Common, MixEnvironment
{
std::vector<std::string> command;
std::optional<std::string> phase;
CmdDevelop()
{
@ -253,6 +278,43 @@ struct CmdDevelop : Common, MixEnvironment
command = ss;
}}
});
addFlag({
.longName = "phase",
.description = "phase to run (e.g. `build` or `configure`)",
.labels = {"phase-name"},
.handler = {&phase},
});
addFlag({
.longName = "configure",
.description = "run the configure phase",
.handler = {&phase, {"configure"}},
});
addFlag({
.longName = "build",
.description = "run the build phase",
.handler = {&phase, {"build"}},
});
addFlag({
.longName = "check",
.description = "run the check phase",
.handler = {&phase, {"check"}},
});
addFlag({
.longName = "install",
.description = "run the install phase",
.handler = {&phase, {"install"}},
});
addFlag({
.longName = "installcheck",
.description = "run the installcheck phase",
.handler = {&phase, {"installCheck"}},
});
}
std::string description() override
@ -288,19 +350,31 @@ struct CmdDevelop : Common, MixEnvironment
auto [rcFileFd, rcFilePath] = createTempFile("nix-shell");
std::ostringstream ss;
makeRcScript(buildEnvironment, ss);
auto script = makeRcScript(buildEnvironment);
ss << fmt("rm -f '%s'\n", rcFilePath);
if (verbosity >= lvlDebug)
script += "set -x\n";
if (!command.empty()) {
script += fmt("rm -f '%s'\n", rcFilePath);
if (phase) {
if (!command.empty())
throw UsageError("you cannot use both '--command' and '--phase'");
// FIXME: foundMakefile is set by buildPhase, need to get
// rid of that.
script += fmt("foundMakefile=1\n");
script += fmt("runHook %1%Phase\n", *phase);
script += fmt("exit 0\n", *phase);
}
else if (!command.empty()) {
std::vector<std::string> args;
for (auto s : command)
args.push_back(shellEscape(s));
ss << fmt("exec %s\n", concatStringsSep(" ", args));
script += fmt("exec %s\n", concatStringsSep(" ", args));
}
writeFull(rcFileFd.get(), ss.str());
writeFull(rcFileFd.get(), script);
stopProgressBar();
@ -362,7 +436,7 @@ struct CmdPrintDevEnv : Common
stopProgressBar();
makeRcScript(buildEnvironment, std::cout);
std::cout << makeRcScript(buildEnvironment);
}
};

17
src/nix/get-env.sh
View file

@ -1,12 +1,6 @@
set -e
if [ -e .attrs.sh ]; then source .attrs.sh; fi
outputs=$_outputs_saved
for __output in $_outputs_saved; do
declare "$__output"="$out"
done
unset _outputs_saved __output
export IN_NIX_SHELL=impure
export dontAddDisableDepTrack=1
@ -14,5 +8,12 @@ if [[ -n $stdenv ]]; then
source $stdenv/setup
fi
export > $out
set >> $out
for __output in $outputs; do
if [[ -z $__done ]]; then
export > ${!__output}
set >> ${!__output}
__done=1
else
echo -n >> ${!__output}
fi
done

28
src/nix/repl.cc
View file

@ -65,7 +65,7 @@ struct NixRepl
void mainLoop(const std::vector<std::string> & files);
StringSet completePrefix(string prefix);
bool getLine(string & input, const std::string &prompt);
Path getDerivationPath(Value & v);
StorePath getDerivationPath(Value & v);
bool processLine(string line);
void loadFile(const Path & path);
void initEnv();
@ -376,13 +376,16 @@ bool isVarName(const string & s)
}
Path NixRepl::getDerivationPath(Value & v) {
StorePath NixRepl::getDerivationPath(Value & v) {
auto drvInfo = getDerivation(*state, v, false);
if (!drvInfo)
throw Error("expression does not evaluate to a derivation, so I can't build it");
Path drvPath = drvInfo->queryDrvPath();
if (drvPath == "" || !state->store->isValidPath(state->store->parseStorePath(drvPath)))
throw Error("expression did not evaluate to a valid derivation");
Path drvPathRaw = drvInfo->queryDrvPath();
if (drvPathRaw == "")
throw Error("expression did not evaluate to a valid derivation (no drv path)");
StorePath drvPath = state->store->parseStorePath(drvPathRaw);
if (!state->store->isValidPath(drvPath))
throw Error("expression did not evaluate to a valid derivation (invalid drv path)");
return drvPath;
}
@ -476,29 +479,30 @@ bool NixRepl::processLine(string line)
evalString("drv: (import <nixpkgs> {}).runCommand \"shell\" { buildInputs = [ drv ]; } \"\"", f);
state->callFunction(f, v, result, Pos());
Path drvPath = getDerivationPath(result);
runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath});
StorePath drvPath = getDerivationPath(result);
runProgram(settings.nixBinDir + "/nix-shell", Strings{state->store->printStorePath(drvPath)});
}
else if (command == ":b" || command == ":i" || command == ":s") {
Value v;
evalString(arg, v);
Path drvPath = getDerivationPath(v);
StorePath drvPath = getDerivationPath(v);
Path drvPathRaw = state->store->printStorePath(drvPath);
if (command == ":b") {
/* We could do the build in this process using buildPaths(),
but doing it in a child makes it easier to recover from
problems / SIGINT. */
if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) {
auto drv = readDerivation(*state->store, drvPath, Derivation::nameFromPath(state->store->parseStorePath(drvPath)));
if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPathRaw}) == 0) {
auto drv = state->store->readDerivation(drvPath);
std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
for (auto & i : drv.outputsAndPaths(*state->store))
std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(i.second.second));
}
} else if (command == ":i") {
runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath});
runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPathRaw});
} else {
runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath});
runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPathRaw});
}
}