Merge pull request #4372 from tweag/ca/drvoutputs-commands

Add a new Cmd type working on RealisedPaths

src/libcmd/command.cc

@@ -54,7 +54,7 @@ void StoreCommand::run()
     run(getStore());
 }
 
-StorePathsCommand::StorePathsCommand(bool recursive)
+RealisedPathsCommand::RealisedPathsCommand(bool recursive)
     : recursive(recursive)
 {
     if (recursive)
@@ -81,30 +81,40 @@ StorePathsCommand::StorePathsCommand(bool recursive)
     });
 }
 
-void StorePathsCommand::run(ref<Store> store)
+void RealisedPathsCommand::run(ref<Store> store)
 {
-    StorePaths storePaths;
+    std::vector<RealisedPath> paths;
-
     if (all) {
         if (installables.size())
             throw UsageError("'--all' does not expect arguments");
+        // XXX: Only uses opaque paths, ignores all the realisations
         for (auto & p : store->queryAllValidPaths())
-            storePaths.push_back(p);
+            paths.push_back(p);
-    }
+    } else {
-
+        auto pathSet = toRealisedPaths(store, realiseMode, operateOn, installables);
-    else {
-        for (auto & p : toStorePaths(store, realiseMode, operateOn, installables))
-            storePaths.push_back(p);
-
         if (recursive) {
-            StorePathSet closure;
+            auto roots = std::move(pathSet);
-            store->computeFSClosure(StorePathSet(storePaths.begin(), storePaths.end()), closure, false, false);
+            pathSet = {};
-            storePaths.clear();
+            RealisedPath::closure(*store, roots, pathSet);
-            for (auto & p : closure)
-                storePaths.push_back(p);
         }
+        for (auto & path : pathSet)
+            paths.push_back(path);
     }
 
+    run(store, std::move(paths));
+}
+
+StorePathsCommand::StorePathsCommand(bool recursive)
+    : RealisedPathsCommand(recursive)
+{
+}
+
+void StorePathsCommand::run(ref<Store> store, std::vector<RealisedPath> paths)
+{
+    StorePaths storePaths;
+    for (auto & p : paths)
+        storePaths.push_back(p.path());
+
     run(store, std::move(storePaths));
 }
 

src/libcmd/command.hh

@@ -141,7 +141,7 @@ private:
 };
 
 /* A command that operates on zero or more store paths. */
-struct StorePathsCommand : public InstallablesCommand
+struct RealisedPathsCommand : public InstallablesCommand
 {
 private:
 
@@ -154,17 +154,28 @@ protected:
 
 public:
 
-    StorePathsCommand(bool recursive = false);
+    RealisedPathsCommand(bool recursive = false);
 
     using StoreCommand::run;
 
-    virtual void run(ref<Store> store, std::vector<StorePath> storePaths) = 0;
+    virtual void run(ref<Store> store, std::vector<RealisedPath> paths) = 0;
 
     void run(ref<Store> store) override;
 
     bool useDefaultInstallables() override { return !all; }
 };
 
+struct StorePathsCommand : public RealisedPathsCommand
+{
+    StorePathsCommand(bool recursive = false);
+
+    using RealisedPathsCommand::run;
+
+    virtual void run(ref<Store> store, std::vector<StorePath> storePaths) = 0;
+
+    void run(ref<Store> store, std::vector<RealisedPath> paths) override;
+};
+
 /* A command that operates on exactly one store path. */
 struct StorePathCommand : public InstallablesCommand
 {
@@ -218,6 +229,12 @@ std::set<StorePath> toDerivations(ref<Store> store,
     std::vector<std::shared_ptr<Installable>> installables,
     bool useDeriver = false);
 
+std::set<RealisedPath> toRealisedPaths(
+    ref<Store> store,
+    Realise mode,
+    OperateOn operateOn,
+    std::vector<std::shared_ptr<Installable>> installables);
+
 /* Helper function to generate args that invoke $EDITOR on
    filename:lineno. */
 Strings editorFor(const Pos & pos);

src/libcmd/installables.cc

@@ -704,23 +704,42 @@ Buildables build(ref<Store> store, Realise mode,
     return buildables;
 }
 
-StorePathSet toStorePaths(ref<Store> store,
+std::set<RealisedPath> toRealisedPaths(
-    Realise mode, OperateOn operateOn,
+    ref<Store> store,
+    Realise mode,
+    OperateOn operateOn,
     std::vector<std::shared_ptr<Installable>> installables)
 {
-    StorePathSet outPaths;
+    std::set<RealisedPath> res;
-
     if (operateOn == OperateOn::Output) {
         for (auto & b : build(store, mode, installables))
             std::visit(overloaded {
                 [&](BuildableOpaque bo) {
-                    outPaths.insert(bo.path);
+                    res.insert(bo.path);
                 },
                 [&](BuildableFromDrv bfd) {
+                    auto drv = store->readDerivation(bfd.drvPath);
+                    auto outputHashes = staticOutputHashes(*store, drv);
                     for (auto & output : bfd.outputs) {
-                        if (!output.second)
+                        if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
-                            throw Error("Cannot operate on output of unbuilt CA drv");
+                            if (!outputHashes.count(output.first))
-                        outPaths.insert(*output.second);
+                                throw Error(
+                                    "the derivation '%s' doesn't have an output named '%s'",
+                                    store->printStorePath(bfd.drvPath),
+                                    output.first);
+                            auto outputId = DrvOutput{outputHashes.at(output.first), output.first};
+                            auto realisation = store->queryRealisation(outputId);
+                            if (!realisation)
+                                throw Error("cannot operate on an output of unbuilt content-addresed derivation '%s'", outputId.to_string());
+                            res.insert(RealisedPath{*realisation});
+                        }
+                        else {
+                            // If ca-derivations isn't enabled, behave as if
+                            // all the paths are opaque to keep the default
+                            // behavior
+                            assert(output.second);
+                            res.insert(*output.second);
+                        }
                     }
                 },
             }, b);
@@ -731,9 +750,19 @@ StorePathSet toStorePaths(ref<Store> store,
         for (auto & i : installables)
             for (auto & b : i->toBuildables())
                 if (auto bfd = std::get_if<BuildableFromDrv>(&b))
-                    outPaths.insert(bfd->drvPath);
+                    res.insert(bfd->drvPath);
     }
 
+    return res;
+}
+
+StorePathSet toStorePaths(ref<Store> store,
+    Realise mode, OperateOn operateOn,
+    std::vector<std::shared_ptr<Installable>> installables)
+{
+    StorePathSet outPaths;
+    for (auto & path : toRealisedPaths(store, mode, operateOn, installables))
+            outPaths.insert(path.path());
     return outPaths;
 }
 

src/libstore/realisation.cc

@@ -46,4 +46,35 @@ Realisation Realisation::fromJSON(
     };
 }
 
+StorePath RealisedPath::path() const {
+    return std::visit([](auto && arg) { return arg.getPath(); }, raw);
+}
+
+void RealisedPath::closure(
+    Store& store,
+    const RealisedPath::Set& startPaths,
+    RealisedPath::Set& ret)
+{
+    // FIXME: This only builds the store-path closure, not the real realisation
+    // closure
+    StorePathSet initialStorePaths, pathsClosure;
+    for (auto& path : startPaths)
+        initialStorePaths.insert(path.path());
+    store.computeFSClosure(initialStorePaths, pathsClosure);
+    ret.insert(startPaths.begin(), startPaths.end());
+    ret.insert(pathsClosure.begin(), pathsClosure.end());
+}
+
+void RealisedPath::closure(Store& store, RealisedPath::Set & ret) const
+{
+    RealisedPath::closure(store, {*this}, ret);
+}
+
+RealisedPath::Set RealisedPath::closure(Store& store) const
+{
+    RealisedPath::Set ret;
+    closure(store, ret);
+    return ret;
+}
+
 } // namespace nix

src/libstore/realisation.hh

@@ -2,6 +2,7 @@
 
 #include "path.hh"
 #include <nlohmann/json_fwd.hpp>
+#include "comparator.hh"
 
 namespace nix {
 
@@ -17,13 +18,7 @@ struct DrvOutput {
 
     static DrvOutput parse(const std::string &);
 
-    bool operator<(const DrvOutput& other) const { return to_pair() < other.to_pair(); }
+    GENERATE_CMP(DrvOutput, me->drvHash, me->outputName);
-    bool operator==(const DrvOutput& other) const { return to_pair() == other.to_pair(); }
-
-private:
-    // Just to make comparison operators easier to write
-    std::pair<Hash, std::string> to_pair() const
-    { return std::make_pair(drvHash, outputName); }
 };
 
 struct Realisation {
@@ -32,8 +27,47 @@ struct Realisation {
 
     nlohmann::json toJSON() const;
     static Realisation fromJSON(const nlohmann::json& json, const std::string& whence);
+
+    StorePath getPath() const { return outPath; }
+
+    GENERATE_CMP(Realisation, me->id, me->outPath);
 };
 
-typedef std::map<DrvOutput, Realisation> DrvOutputs;
+struct OpaquePath {
+    StorePath path;
+
+    StorePath getPath() const { return path; }
+
+    GENERATE_CMP(OpaquePath, me->path);
+};
+
+
+/**
+ * A store path with all the history of how it went into the store
+ */
+struct RealisedPath {
+    /*
+     * A path is either the result of the realisation of a derivation or
+     * an opaque blob that has been directly added to the store
+     */
+    using Raw = std::variant<Realisation, OpaquePath>;
+    Raw raw;
+
+    using Set = std::set<RealisedPath>;
+
+    RealisedPath(StorePath path) : raw(OpaquePath{path}) {}
+    RealisedPath(Realisation r) : raw(r) {}
+
+    /**
+     * Get the raw store path associated to this
+     */
+    StorePath path() const;
+
+    void closure(Store& store, Set& ret) const;
+    static void closure(Store& store, const Set& startPaths, Set& ret);
+    Set closure(Store& store) const;
+
+    GENERATE_CMP(RealisedPath, me->raw);
+};
 
 }

src/libutil/comparator.hh

@@ -0,0 +1,30 @@
+#pragma once
+
+/* Awfull hacky generation of the comparison operators by doing a lexicographic
+ * comparison between the choosen fields.
+ *
+ * ```
+ * GENERATE_CMP(ClassName, me->field1, me->field2, ...)
+ * ```
+ *
+ * will generate comparison operators semantically equivalent to:
+ *
+ * ```
+ * bool operator<(const ClassName& other) {
+ *   return field1 < other.field1 && field2 < other.field2 && ...;
+ * }
+ * ```
+ */
+#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, FIELDS...) \
+    bool operator COMPARATOR(const MY_TYPE& other) const { \
+      const MY_TYPE* me = this; \
+      auto fields1 = std::make_tuple( FIELDS ); \
+      me = &other; \
+      auto fields2 = std::make_tuple( FIELDS ); \
+      return fields1 COMPARATOR fields2; \
+    }
+#define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args)
+#define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args)
+#define GENERATE_CMP(args...) \
+    GENERATE_EQUAL(args) \
+    GENERATE_LEQ(args)

src/nix/copy.cc

@@ -16,6 +16,8 @@ struct CmdCopy : StorePathsCommand
 
     SubstituteFlag substitute = NoSubstitute;
 
+    using StorePathsCommand::run;
+
     CmdCopy()
         : StorePathsCommand(true)
     {