Allow enabling core dumps from builds for nix & child processes

Fixes https://git.lix.systems/lix-project/lix/issues/268

Change-Id: I3f1b0ddf064f891cca8b53229c5c31c74cea3d9f

doc/manual/change-authors.yml

@@ -61,3 +61,8 @@ edolstra:
 roberth:
   display_name: Robert Hensing
   github: roberth
+
+midnightveil:
+  display_name: julia
+  forgejo: midnightveil
+  github: midnightveil

doc/manual/rl-next/enable-coredumps.md

@@ -0,0 +1,13 @@
+---
+synopsis: "Add an option `enable-core-dumps` that enables core dumps from builds"
+cls: 1088
+credits: midnightveil
+category: Features
+---
+
+In the past, Lix disabled core dumps by setting the soft `RLIMIT_CORE` to 0
+unconditionally. Although this rlimit could be altered from the builder since
+it is just the soft limit, this was kind of annoying to do. By passing
+`--option enable-core-dumps true` to an offending build, one can now cause the
+core dumps to be handled by the system in the normal way (winding up in
+`coredumpctl`, say, on Linux).

src/libstore/build/local-derivation-goal.cc

@@ -1977,6 +1977,9 @@ void LocalDerivationGoal::runChild()
 
         /* Disable core dumps by default. */
         struct rlimit limit = { 0, RLIM_INFINITY };
+        if (settings.enableCoreDumps) {
+            limit.rlim_cur = RLIM_INFINITY;
+        }
         setrlimit(RLIMIT_CORE, &limit);
 
         // FIXME: set other limits to deterministic values?

src/libstore/globals.hh

@@ -290,6 +290,15 @@ public:
     Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space",
         "Amount of reserved disk space for the garbage collector."};
 
+    Setting<bool> enableCoreDumps{
+        this, false, "enable-core-dumps",
+        R"(
+          If set to `false` (the default), `RLIMIT_CORE` has a soft limit of zero.
+          If set to `true`, the soft limit is infinite.
+
+          The hard limit is always infinite.
+        )"};
+
     Setting<bool> fsyncMetadata{
         this, true, "fsync-metadata",
         R"(

tests/nixos/coredumps/default.nix

@@ -0,0 +1,15 @@
+let
+  inherit (import ../util.nix) mkNixBuildTest;
+in mkNixBuildTest rec {
+  name = "coredumps";
+  extraMachineConfig = { pkgs, ... }: {
+    boot.kernel.sysctl."kernel.core_pattern" = "core";
+  };
+
+  expressionFile = ./package.nix;
+
+  testScriptPost = ''
+    # do a test, but this time with coredumps enabled.
+    machine.succeed('nix-build --option enable-core-dumps true --expr "let pkgs = import <nixpkgs> {}; in pkgs.callPackage ${expressionFile} { shouldBePresent = true; }"')
+  '';
+}

tests/nixos/coredumps/package.nix

@@ -0,0 +1,16 @@
+{ lib, runCommand, shouldBePresent ? false }:
+
+runCommand "core-dump-now" { } ''
+  set -m
+  sleep infinity &
+
+  # make a coredump
+  kill -SIGSEGV %1
+
+  if ${lib.optionalString (shouldBePresent) "!"} test -n "$(find . -maxdepth 1 -name 'core*' -print -quit)"; then
+    echo "core file was in wrong presence state, expected: ${if shouldBePresent then "present" else "missing"}"
+    exit 1
+  fi
+
+  touch $out
+''

tests/nixos/default.nix

@@ -166,4 +166,6 @@ in
   rootInSandbox = runNixOSTestFor "x86_64-linux" ./root-in-sandbox;
 
   broken-userns = runNixOSTestFor "x86_64-linux" ./broken-userns.nix;
+
+  coredumps = runNixOSTestFor "x86_64-linux" ./coredumps;
 }

tests/nixos/util.nix

@@ -1,5 +1,6 @@
 {
-  mkNixBuildTest = { name, expressionFile, extraMachineConfig ? {} }:
+  mkNixBuildTest =
+    { name, expressionFile, extraMachineConfig ? {}, testScriptPre ? "", testScriptPost ? "" }:
     { lib, pkgs, ... }:
     {
       inherit name;
@@ -17,7 +18,11 @@
       testScript = { nodes }: ''
         start_all()
 
+        ${testScriptPre}
+
         machine.succeed('nix-build --expr "let pkgs = import <nixpkgs> {}; in pkgs.callPackage ${expressionFile} {}"')
+
+        ${testScriptPost}
       '';
     };
 }