Add repl-overlays

Adds a `repl-overlays` option, which specifies files that can overlay
and modify the top-level bindings in `nix repl`. For example, with the
following contents in `~/.config/nix/repl.nix`:

    info: final: prev: let
      optionalAttrs = predicate: attrs:
        if predicate
        then attrs
        else {};
    in
      optionalAttrs (prev ? legacyPackages && prev.legacyPackages ? ${info.currentSystem})
      {
        pkgs = prev.legacyPackages.${info.currentSystem};
      }

We can run `nix repl` and use `pkgs` to refer to `legacyPackages.${currentSystem}`:

    $ nix repl --repl-overlays ~/.config/nix/repl.nix nixpkgs
    Nix 2.21.0pre20240309_4111bb6
    Type :? for help.
    Loading installable 'flake:nixpkgs#'...
    Added 5 variables.
    Loading 'repl-overlays'...
    Added 6 variables.
    nix-repl> pkgs.bash
    «derivation /nix/store/g08b5vkwwh0j8ic9rkmd8mpj878rk62z-bash-5.2p26.drv»

Change-Id: Ic12e0f2f210b2f46e920c33088dfe1083f42391a

doc/manual/rl-next/repl-overlays.md

@@ -0,0 +1,35 @@
+---
+synopsis: Add `repl-overlays` option
+prs: 10203
+---
+
+A `repl-overlays` option has been added, which specifies files that can overlay
+and modify the top-level bindings in `nix repl`. For example, with the
+following contents in `~/.config/nix/repl.nix`:
+
+```nix
+info: final: prev: let
+  optionalAttrs = predicate: attrs:
+    if predicate
+    then attrs
+    else {};
+in
+  optionalAttrs (prev ? legacyPackages && prev.legacyPackages ? ${info.currentSystem})
+  {
+    pkgs = prev.legacyPackages.${info.currentSystem};
+  }
+```
+
+We can run `nix repl` and use `pkgs` to refer to `legacyPackages.${currentSystem}`:
+
+```ShellSession
+$ nix repl --repl-overlays ~/.config/nix/repl.nix nixpkgs
+Nix 2.21.0pre20240309_4111bb6
+Type :? for help.
+Loading installable 'flake:nixpkgs#'...
+Added 5 variables.
+Loading 'repl-overlays'...
+Added 6 variables.
+nix-repl> pkgs.bash
+«derivation /nix/store/g08b5vkwwh0j8ic9rkmd8mpj878rk62z-bash-5.2p26.drv»
+```

src/libcmd/local.mk

@@ -13,3 +13,5 @@ libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread
 libcmd_LIBS = libstore libutil libexpr libmain libfetchers
 
 $(eval $(call install-file-in, $(buildprefix)$(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644))
+
+$(d)/repl.cc: $(d)/repl-overlays.nix.gen.hh

src/libcmd/meson.build

@@ -1,3 +1,15 @@
+libcmd_generated_headers = []
+foreach header : [ 'repl-overlays.nix' ]
+  libcmd_generated_headers += custom_target(
+    command : [ 'bash', '-c', 'echo \'R"__NIX_STR(\' | cat - @INPUT@ && echo \')__NIX_STR"\'' ],
+    input : header,
+    output : '@PLAINNAME@.gen.hh',
+    capture : true,
+    install : true,
+    install_dir : includedir / 'nix',
+  )
+endforeach
+
 libcmd_sources = files(
   'built-path.cc',
   'command-installable-value.cc',
@@ -34,6 +46,7 @@ libcmd_headers = files(
 
 libcmd = library(
   'nixcmd',
+  libcmd_generated_headers,
   libcmd_sources,
   dependencies : [
     liblixutil,

src/libcmd/repl-overlays.nix

@@ -0,0 +1,8 @@
+info:
+initial:
+functions:
+let final = builtins.foldl'
+              (prev: function: prev // (function info final prev))
+              initial
+              functions;
+in final

src/libcmd/repl.cc

@@ -30,6 +30,7 @@
 #include "signals.hh"
 #include "print.hh"
 #include "progress-bar.hh"
+#include "gc-small-vector.hh"
 
 #if HAVE_BOEHMGC
 #define GC_INCLUDE_NEW
@@ -100,6 +101,45 @@ struct NixRepl
     void evalString(std::string s, Value & v);
     void loadDebugTraceEnv(DebugTrace & dt);
 
+    /**
+     * Load the `repl-overlays` and add the resulting AttrSet to the top-level
+     * bindings.
+     */
+    void loadReplOverlays();
+
+    /**
+     * Get a list of each of the `repl-overlays` (parsed and evaluated).
+     */
+    Value * replOverlays();
+
+    /**
+     * Get the Nix function that composes the `repl-overlays` together.
+     */
+    Value * getReplOverlaysEvalFunction();
+
+    /**
+     * Cached return value of `getReplOverlaysEvalFunction`.
+     *
+     * Note: This is `shared_ptr` to avoid garbage collection.
+     */
+    std::shared_ptr<Value *> replOverlaysEvalFunction =
+        std::allocate_shared<Value *>(traceable_allocator<Value *>(), nullptr);
+
+    /**
+     * Get the `info` AttrSet that's passed as the first argument to each
+     * of the `repl-overlays`.
+     */
+    Value * replInitInfo();
+
+    /**
+     * Get the current top-level bindings as an AttrSet.
+     */
+    Value * bindingsToAttrs();
+    /**
+     * Parse a file, evaluate its result, and force the resulting value.
+     */
+    Value * evalFile(SourcePath & path);
+
     void printValue(std::ostream & str,
                               Value & v,
                               unsigned int maxDepth = std::numeric_limits<unsigned int>::max())
@@ -737,14 +777,119 @@ void NixRepl::loadFiles()
     loadedFiles.clear();
 
     for (auto & i : old) {
-        notice("Loading '%1%'...", i);
+        notice("Loading '%1%'...", Magenta(i));
         loadFile(i);
     }
 
     for (auto & [i, what] : getValues()) {
-        notice("Loading installable '%1%'...", what);
+        notice("Loading installable '%1%'...", Magenta(what));
         addAttrsToScope(*i);
     }
+
+    loadReplOverlays();
+}
+
+void NixRepl::loadReplOverlays()
+{
+    if (!evalSettings.replOverlays) {
+        return;
+    }
+
+    notice("Loading '%1%'...", Magenta("repl-overlays"));
+    auto replInitFilesFunction = getReplOverlaysEvalFunction();
+
+    Value &newAttrs(*state->allocValue());
+    SmallValueVector<3> args = {replInitInfo(), bindingsToAttrs(), replOverlays()};
+    state->callFunction(
+        *replInitFilesFunction,
+        args.size(),
+        args.data(),
+        newAttrs,
+        replInitFilesFunction->determinePos(noPos)
+    );
+
+    addAttrsToScope(newAttrs);
+}
+
+Value * NixRepl::getReplOverlaysEvalFunction()
+{
+    if (replOverlaysEvalFunction && *replOverlaysEvalFunction) {
+        return *replOverlaysEvalFunction;
+    }
+
+    auto evalReplInitFilesPath = CanonPath::root + "repl-overlays.nix";
+    *replOverlaysEvalFunction = state->allocValue();
+    auto code =
+        #include "repl-overlays.nix.gen.hh"
+        ;
+    auto expr = state->parseExprFromString(
+        code,
+        SourcePath(evalReplInitFilesPath),
+        state->staticBaseEnv
+    );
+
+    state->eval(expr, **replOverlaysEvalFunction);
+    state->forceValue(**replOverlaysEvalFunction, (*replOverlaysEvalFunction)->determinePos(noPos));
+
+    return *replOverlaysEvalFunction;
+}
+
+Value * NixRepl::replOverlays()
+{
+    Value * replInits(state->allocValue());
+    state->mkList(*replInits, evalSettings.replOverlays.get().size());
+    Value ** replInitElems = replInits->listElems();
+
+    size_t i = 0;
+    for (auto path : evalSettings.replOverlays.get()) {
+        debug("Loading '%1%' path '%2%'...", "repl-overlays", path);
+        SourcePath sourcePath((CanonPath(path)));
+        auto replInit = evalFile(sourcePath);
+
+        if (!replInit->isLambda()) {
+            state->error<TypeError>(
+                "Expected `repl-overlays` to be a lambda but found %1%: %2%",
+                showType(*replInit),
+                ValuePrinter(*state, *replInit, errorPrintOptions)
+            )
+                .atPos(replInit->determinePos(noPos))
+                .debugThrow();
+        }
+
+        if (replInit->lambda.fun->hasFormals()
+            && !replInit->lambda.fun->formals->ellipsis) {
+            state->error<TypeError>(
+                "Expected first argument of %1% to have %2% to allow future versions of Lix to add additional attributes to the argument",
+                "repl-overlays",
+                "..."
+            )
+                .atPos(replInit->determinePos(noPos))
+                .debugThrow();
+        }
+
+        replInitElems[i] = replInit;
+        i++;
+    }
+
+
+    return replInits;
+}
+
+Value * NixRepl::replInitInfo()
+{
+    auto builder = state->buildBindings(2);
+
+    Value * currentSystem(state->allocValue());
+    currentSystem->mkString(evalSettings.getCurrentSystem());
+    builder.insert(state->symbols.create("currentSystem"), currentSystem);
+
+    Value * valueNull(state->allocValue());
+    valueNull->mkNull();
+    builder.insert(state->symbols.create("__pleaseUseDotDotDot"), valueNull);
+
+    Value * info(state->allocValue());
+    info->mkAttrs(builder.finish());
+    return info;
 }
 
 
@@ -777,6 +922,18 @@ void NixRepl::addVarToScope(const Symbol name, Value & v)
     varNames.emplace(state->symbols[name]);
 }
 
+Value * NixRepl::bindingsToAttrs()
+{
+    auto builder = state->buildBindings(staticEnv->vars.size());
+    for (auto & [symbol, displacement] : staticEnv->vars) {
+        builder.insert(symbol, env->values[displacement]);
+    }
+
+    Value * attrs(state->allocValue());
+    attrs->mkAttrs(builder.finish());
+    return attrs;
+}
+
 
 Expr * NixRepl::parseString(std::string s)
 {
@@ -791,6 +948,15 @@ void NixRepl::evalString(std::string s, Value & v)
     state->forceValue(v, v.determinePos(noPos));
 }
 
+Value * NixRepl::evalFile(SourcePath & path)
+{
+    auto expr = state->parseExprFromFile(path, staticEnv);
+    Value * result(state->allocValue());
+    expr->eval(*state, *env, *result);
+    state->forceValue(*result, result->determinePos(noPos));
+    return result;
+}
+
 
 std::unique_ptr<AbstractNixRepl> AbstractNixRepl::create(
    const SearchPath & searchPath, nix::ref<Store> store, ref<EvalState> state,

src/libexpr/eval-settings.hh

@@ -124,6 +124,42 @@ struct EvalSettings : Config
 
           This is useful for debugging warnings in third-party Nix code.
         )"};
+
+    PathsSetting replOverlays{this, Paths(), "repl-overlays",
+        R"(
+          A list of files containing Nix expressions that can be used to add
+          default bindings to [`nix
+          repl`](@docroot@/command-ref/new-cli/nix3-repl.md) sessions.
+
+          Each file is called with three arguments:
+          1. An [attribute set](@docroot@/language/values.html#attribute-set)
+             containing at least a
+             [`currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem)
+             attribute (this is identical to
+             [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem),
+             except that it's available in
+             [`pure-eval`](@docroot@/command-ref/conf-file.html#conf-pure-eval)
+             mode).
+          2. The top-level bindings produced by the previous `repl-overlays`
+             value (or the default top-level bindings).
+          3. The final top-level bindings produced by calling all
+             `repl-overlays`.
+
+          For example, the following file would alias `pkgs` to
+          `legacyPackages.${info.currentSystem}` (if that attribute is defined):
+
+          ```nix
+          info: prev: final:
+          if prev ? legacyPackages
+             && prev.legacyPackages ? ${info.currentSystem}
+          then
+          {
+            pkgs = prev.legacyPackages.${info.currentSystem};
+          }
+          else
+          { }
+          ```
+        )"};
 };
 
 extern EvalSettings evalSettings;

tests/functional/repl_characterization/data/extra_data/repl-overlay-fail.nix

@@ -0,0 +1 @@
+info: final: prev: builtins.abort "uh oh!"

tests/functional/repl_characterization/data/extra_data/repl-overlay-no-dotdotdot.nix

@@ -0,0 +1,6 @@
+let
+  puppy = "doggy";
+in
+  {currentSystem}: final: prev: {
+    inherit puppy;
+  }

tests/functional/repl_characterization/data/extra_data/repl-overlay-no-formals.nix

@@ -0,0 +1 @@
+info: final: prev: {}

tests/functional/repl_characterization/data/extra_data/repl-overlay-packages-is-pkgs.nix

@@ -0,0 +1,4 @@
+info: final: prev:
+{
+  pkgs = final.packages.x86_64-linux;
+}

tests/functional/repl_characterization/data/extra_data/repl-overlays-compose-1.nix

@@ -0,0 +1,7 @@
+info: final: prev:
+{
+  var = prev.var + "b";
+
+  # We can access the final value of `var` here even though it isn't defined yet:
+  varUsingFinal = "final value is: " + final.newVar;
+}

tests/functional/repl_characterization/data/extra_data/repl-overlays-compose-2.nix

@@ -0,0 +1,6 @@
+info: final: prev:
+{
+  var = prev.var + "c";
+
+  newVar = "puppy";
+}

tests/functional/repl_characterization/data/repl_overlays.nix

@@ -0,0 +1,3 @@
+{
+  packages.x86_64-linux.default = "my package";
+}

tests/functional/repl_characterization/data/repl_overlays.test

@@ -0,0 +1,5 @@
+Check basic `repl-overlays` functionality.
+@args --repl-overlays
+@args ${PWD}/extra_data/repl-overlay-packages-is-pkgs.nix
+  nix-repl> pkgs
+  { default = "my package"; }

tests/functional/repl_characterization/data/repl_overlays_compose.nix

@@ -0,0 +1,3 @@
+{
+  var = "a";
+}

tests/functional/repl_characterization/data/repl_overlays_compose.test

@@ -0,0 +1,7 @@
+Check that multiple `repl-overlays` can compose together
+@args --repl-overlays
+@args "${PWD}/extra_data/repl-overlays-compose-1.nix ${PWD}/extra_data/repl-overlays-compose-2.nix"
+  nix-repl> var
+  "abc"
+  nix-repl> varUsingFinal
+  "final value is: puppy"

tests/functional/repl_characterization/data/repl_overlays_destructure_without_dotdotdot_errors.test

@@ -0,0 +1,10 @@
+`repl-overlays` that try to parse out the `info` argument without a `...` error.
+@args --repl-overlays
+@args ${PWD}/extra_data/repl-overlay-no-dotdotdot.nix
+@should-start false
+  error: Expected first argument of repl-overlays to have ... to allow future versions of Lix to add additional attributes to the argument
+         at $TEST_DATA/extra_data/repl-overlay-no-dotdotdot.nix:4:3:
+              3| in
+              4|   {currentSystem}: final: prev: {
+               |   ^
+              5|     inherit puppy;\n

tests/functional/repl_characterization/data/repl_overlays_destructure_without_formals_ok.test

@@ -0,0 +1,5 @@
+`repl-overlays` that don't destructure the `info` argument are OK.
+@args --repl-overlays
+@args ${PWD}/extra_data/repl-overlay-no-formals.nix
+  nix-repl> 1
+  1

tests/functional/repl_characterization/data/repl_overlays_error.test

@@ -0,0 +1,22 @@
+`repl-overlays` that fail to evaluate should error.
+@args --repl-overlays
+@args ${PWD}/extra_data/repl-overlay-fail.nix
+@should-start false
+  error:
+         … while calling the 'foldl'' builtin
+           at «string»:5:13:
+              4| functions:
+              5| let final = builtins.foldl'
+               |             ^
+              6|               (prev: function: prev // (function info final prev))
+
+         … in the right operand of the update (//) operator
+           at «string»:6:37:
+              5| let final = builtins.foldl'
+              6|               (prev: function: prev // (function info final prev))
+               |                                     ^
+              7|               initial
+
+         (stack trace truncated; use '--show-trace' to show the full trace)
+
+         error: evaluation aborted with the following error message: 'uh oh!'

tests/functional/repl_characterization/repl_characterization.cc

@@ -179,9 +179,16 @@ TEST_F(ReplSessionTest, tidy)
         runReplTestPath(#name);   \
     }
 
-REPL_TEST(regression_9918);
-REPL_TEST(regression_9917);
-REPL_TEST(stack_vars);
 REPL_TEST(basic_repl);
+REPL_TEST(basic_tidied);
+REPL_TEST(regression_9917);
+REPL_TEST(regression_9918);
+REPL_TEST(regression_l145);
+REPL_TEST(repl_overlays);
+REPL_TEST(repl_overlays_compose);
+REPL_TEST(repl_overlays_destructure_without_dotdotdot_errors);
+REPL_TEST(repl_overlays_destructure_without_formals_ok);
+REPL_TEST(repl_overlays_error);
+REPL_TEST(stack_vars);
 
 }; // namespace nix