* Refactorings.

src/Makefile.am

@@ -20,7 +20,8 @@ test_LDADD = libshared.a libnix.a -ldb_cxx-4 -lATerm
 noinst_LIBRARIES = libnix.a libshared.a
 
 libnix_a_SOURCES = util.cc hash.cc archive.cc md5.c \
- fstate.cc store.cc globals.cc db.cc references.cc
+ store.cc fstate.cc normalise.cc exec.cc \
+ globals.cc db.cc references.cc
 
 libshared_a_SOURCES = shared.cc
 

src/exec.cc

@@ -0,0 +1,123 @@
+#include <iostream>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+#include "exec.hh"
+#include "util.hh"
+#include "globals.hh"
+
+
+class AutoDelete
+{
+    string path;
+    bool del;
+public:
+
+    AutoDelete(const string & p) : path(p) 
+    {
+        del = true;
+    }
+
+    ~AutoDelete()
+    {
+        if (del) deletePath(path);
+    }
+
+    void cancel()
+    {
+        del = false;
+    }
+};
+
+
+/* Run a program. */
+void runProgram(const string & program, Environment env)
+{
+    /* Create a log file. */
+    string logFileName = nixLogDir + "/run.log";
+    /* !!! auto-pclose on exit */
+    FILE * logFile = popen(("tee -a " + logFileName + " >&2").c_str(), "w"); /* !!! escaping */
+    if (!logFile)
+        throw SysError(format("creating log file `%1%'") % logFileName);
+
+    /* Create a temporary directory where the build will take
+       place. */
+    static int counter = 0;
+    string tmpDir = (format("/tmp/nix-%1%-%2%") % getpid() % counter++).str();
+
+    if (mkdir(tmpDir.c_str(), 0777) == -1)
+        throw SysError(format("creating directory `%1%'") % tmpDir);
+
+    AutoDelete delTmpDir(tmpDir);
+
+    /* Fork a child to build the package. */
+    pid_t pid;
+    switch (pid = fork()) {
+            
+    case -1:
+        throw SysError("unable to fork");
+
+    case 0: 
+
+        try { /* child */
+
+            if (chdir(tmpDir.c_str()) == -1)
+                throw SysError(format("changing into to `%1%'") % tmpDir);
+
+            /* Fill in the environment.  We don't bother freeing
+               the strings, since we'll exec or die soon
+               anyway. */
+            const char * env2[env.size() + 1];
+            int i = 0;
+            for (Environment::iterator it = env.begin();
+                 it != env.end(); it++, i++)
+                env2[i] = (new string(it->first + "=" + it->second))->c_str();
+            env2[i] = 0;
+
+            /* Dup the log handle into stderr. */
+            if (dup2(fileno(logFile), STDERR_FILENO) == -1)
+                throw SysError("cannot pipe standard error into log file");
+            
+            /* Dup stderr to stdin. */
+            if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1)
+                throw SysError("cannot dup stderr into stdout");
+
+            /* Make the program executable.  !!! hack. */
+            if (chmod(program.c_str(), 0755))
+                throw SysError("cannot make program executable");
+
+            /* Execute the program.  This should not return. */
+            execle(program.c_str(), baseNameOf(program).c_str(), 0, env2);
+
+            throw SysError(format("unable to execute %1%") % program);
+            
+        } catch (exception & e) {
+            cerr << format("build error: %1%\n") % e.what();
+        }
+        _exit(1);
+
+    }
+
+    /* parent */
+
+    /* Close the logging pipe.  Note that this should not cause
+       the logger to exit until builder exits (because the latter
+       has an open file handle to the former). */
+    pclose(logFile);
+    
+    /* Wait for the child to finish. */
+    int status;
+    if (waitpid(pid, &status, 0) != pid)
+        throw Error("unable to wait for child");
+    
+    if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
+        delTmpDir.cancel();
+        throw Error("unable to build package");
+    }
+}
+
+

src/exec.hh

@@ -0,0 +1,18 @@
+#ifndef __EXEC_H
+#define __EXEC_H
+
+#include <string>
+#include <map>
+
+using namespace std;
+
+
+/* A Unix environment is a mapping from strings to strings. */
+typedef map<string, string> Environment;
+
+
+/* Run a program. */
+void runProgram(const string & program, Environment env);
+
+
+#endif /* !__EXEC_H */

src/fix.cc

@@ -2,8 +2,7 @@
 #include <iostream>
 
 #include "globals.hh"
-#include "fstate.hh"
+#include "normalise.hh"
-#include "store.hh"
 #include "shared.hh"
 
 
@@ -111,12 +110,11 @@ static Expr evalExpr(Expr e)
         ATmatch(e, "FSId(<str>)", &s1))
         return e;
 
-    if (ATgetType(e) == AT_APPL && 
+    try {
-        ((string) ATgetName(ATgetAFun(e)) == "Slice" ||
+        parseFState(e);
-         (string) ATgetName(ATgetAFun(e)) == "Derive"))
-    {
         return ATmake("FSId(<str>)", 
-            ((string) writeTerm(e, "", 0)).c_str());
+            ((string) writeTerm(e, "")).c_str());
+    } catch (...) { /* !!! catch parse errors only */
     }
 
     /* Application. */
@@ -139,10 +137,17 @@ static Expr evalExpr(Expr e)
         string dstPath;
         FSId id;
         addToStore(srcPath, dstPath, id, true);
-        FState fs = ATmake("Slice([<str>], [(<str>, <str>, [])])",
+
-            ((string) id).c_str(), dstPath.c_str(), ((string) id).c_str());
+        SliceElem elem;
+        elem.path = dstPath;
+        elem.id = id;
+        FState fs;
+        fs.type = FState::fsSlice;
+        fs.slice.roots.push_back(id);
+        fs.slice.elems.push_back(elem);
+
         return ATmake("FSId(<str>)", 
-            ((string) writeTerm(fs, "", 0)).c_str());
+            ((string) writeTerm(unparseFState(fs), "")).c_str());
     }
 
     /* Packages are transformed into Derive fstate expressions. */
@@ -160,8 +165,10 @@ static Expr evalExpr(Expr e)
         }
 
         /* Gather information for building the Derive expression. */
-        ATermList ins = ATempty, env = ATempty;
+        FState fs;
-        string builder, name;
+        fs.type = FState::fsDerive;
+        fs.derive.platform = SYSTEM;
+        string name;
         bnds = ATempty;
 
         for (map<string, ATerm>::iterator it = bndMap.begin();
@@ -169,21 +176,19 @@ static Expr evalExpr(Expr e)
         {
             string key = it->first;
             ATerm value = it->second;
-            char * id;
 
-            if (ATmatch(value, "FSId(<str>)", &id)) {
+            if (ATmatch(value, "FSId(<str>)", &s1)) {
-                Strings paths = fstatePaths(parseHash(id), false);
+                FSId id = parseHash(s1);
+                Strings paths = fstatePaths(id, false);
                 if (paths.size() != 1) abort();
                 string path = *(paths.begin());
-                ins = ATinsert(ins, ATmake("<str>", id));
+                fs.derive.inputs.push_back(id);
-                env = ATinsert(env, ATmake("(<str>, <str>)",
+                fs.derive.env.push_back(StringPair(key, path));
-                    key.c_str(), path.c_str()));
+                if (key == "build") fs.derive.builder = path;
-                if (key == "build") builder = path;
             }
             else if (ATmatch(value, "<str>", &s1)) {
                 if (key == "name") name = s1;
-                env = ATinsert(env, 
+                fs.derive.env.push_back(StringPair(key, s1));
-                    ATmake("(<str>, <str>)", key.c_str(), s1));
             }
             else throw badTerm("invalid package argument", value);
 
@@ -191,31 +196,24 @@ static Expr evalExpr(Expr e)
                 ATmake("(<str>, <term>)", key.c_str(), value));
         }
 
-        /* Hash the normal form to produce a unique but deterministic
+        if (fs.derive.builder == "")
-           path name for this package. */
+            throw badTerm("no builder specified", e);
-        ATerm nf = ATmake("Package(<term>)", ATreverse(bnds));
-        FSId outId = hashTerm(nf);
-
-        if (builder == "")
-            throw badTerm("no builder specified", nf);
         
         if (name == "")
-            throw badTerm("no package name specified", nf);
+            throw badTerm("no package name specified", e);
         
+        /* Hash the fstate-expression with no outputs to produce a
+           unique but deterministic path name for this package. */
+        Hash outId = hashTerm(unparseFState(fs));
         string outPath = 
             canonPath(nixStore + "/" + ((string) outId).c_str() + "-" + name);
-
+        fs.derive.env.push_back(StringPair("out", outPath));
-        env = ATinsert(env, ATmake("(<str>, <str>)", "out", outPath.c_str()));
+        fs.derive.outputs.push_back(DeriveOutput(outPath, outId));
-        
+        debug(format("%1%: %2%") % (string) outId % name);
-        /* Construct the result. */
-        FState fs = 
-            ATmake("Derive([(<str>, <str>)], <term>, <str>, <str>, <term>)",
-                outPath.c_str(), ((string) outId).c_str(),
-                ins, builder.c_str(), SYSTEM, env);
 
         /* Write the resulting term into the Nix store directory. */
         return ATmake("FSId(<str>)", 
-            ((string) writeTerm(fs, "-d-" + name, 0)).c_str());
+            ((string) writeTerm(unparseFState(fs), "-d-" + name)).c_str());
     }
 
     /* BaseName primitive function. */

src/fstate.cc

@@ -1,141 +1,6 @@
-#include <map>
-#include <iostream>
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/wait.h>
-#include <unistd.h>
-#include <fcntl.h>
-
 #include "fstate.hh"
 #include "globals.hh"
 #include "store.hh"
-#include "db.hh"
-#include "references.hh"
-
-
-/* A Unix environment is a mapping from strings to strings. */
-typedef map<string, string> Environment;
-
-
-class AutoDelete
-{
-    string path;
-    bool del;
-public:
-
-    AutoDelete(const string & p) : path(p) 
-    {
-        del = true;
-    }
-
-    ~AutoDelete()
-    {
-        if (del) deletePath(path);
-    }
-
-    void cancel()
-    {
-        del = false;
-    }
-};
-
-
-/* Run a program. */
-static void runProgram(const string & program, Environment env)
-{
-    /* Create a log file. */
-    string logFileName = nixLogDir + "/run.log";
-    /* !!! auto-pclose on exit */
-    FILE * logFile = popen(("tee -a " + logFileName + " >&2").c_str(), "w"); /* !!! escaping */
-    if (!logFile)
-        throw SysError(format("creating log file `%1%'") % logFileName);
-
-    /* Create a temporary directory where the build will take
-       place. */
-    static int counter = 0;
-    string tmpDir = (format("/tmp/nix-%1%-%2%") % getpid() % counter++).str();
-
-    if (mkdir(tmpDir.c_str(), 0777) == -1)
-        throw SysError(format("creating directory `%1%'") % tmpDir);
-
-    AutoDelete delTmpDir(tmpDir);
-
-    /* Fork a child to build the package. */
-    pid_t pid;
-    switch (pid = fork()) {
-            
-    case -1:
-        throw SysError("unable to fork");
-
-    case 0: 
-
-        try { /* child */
-
-            if (chdir(tmpDir.c_str()) == -1)
-                throw SysError(format("changing into to `%1%'") % tmpDir);
-
-            /* Fill in the environment.  We don't bother freeing
-               the strings, since we'll exec or die soon
-               anyway. */
-            const char * env2[env.size() + 1];
-            int i = 0;
-            for (Environment::iterator it = env.begin();
-                 it != env.end(); it++, i++)
-                env2[i] = (new string(it->first + "=" + it->second))->c_str();
-            env2[i] = 0;
-
-            /* Dup the log handle into stderr. */
-            if (dup2(fileno(logFile), STDERR_FILENO) == -1)
-                throw SysError("cannot pipe standard error into log file");
-            
-            /* Dup stderr to stdin. */
-            if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1)
-                throw SysError("cannot dup stderr into stdout");
-
-            /* Make the program executable.  !!! hack. */
-            if (chmod(program.c_str(), 0755))
-                throw SysError("cannot make program executable");
-
-            /* Execute the program.  This should not return. */
-            execle(program.c_str(), baseNameOf(program).c_str(), 0, env2);
-
-            throw SysError(format("unable to execute %1%") % program);
-            
-        } catch (exception & e) {
-            cerr << format("build error: %1%\n") % e.what();
-        }
-        _exit(1);
-
-    }
-
-    /* parent */
-
-    /* Close the logging pipe.  Note that this should not cause
-       the logger to exit until builder exits (because the latter
-       has an open file handle to the former). */
-    pclose(logFile);
-    
-    /* Wait for the child to finish. */
-    int status;
-    if (waitpid(pid, &status, 0) != pid)
-        throw Error("unable to wait for child");
-    
-    if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
-        delTmpDir.cancel();
-        throw Error("unable to build package");
-    }
-}
-
-
-/* Throw an exception if the given platform string is not supported by
-   the platform we are executing on. */
-static void checkPlatform(const string & platform)
-{
-    if (platform != thisSystem)
-        throw Error(format("a `%1%' is required, but I am a `%2%'")
-            % platform % thisSystem);
-}
 
 
 string printTerm(ATerm t)
@@ -157,23 +22,16 @@ Hash hashTerm(ATerm t)
 }
 
 
-FState hash2fstate(Hash hash)
+ATerm termFromId(const FSId & id)
-{
-    return ATmake("Include(<str>)", ((string) hash).c_str());
-}
-
-
-ATerm termFromId(const FSId & id, string * p)
 {
     string path = expandId(id);
-    if (p) *p = path;
     ATerm t = ATreadFromNamedFile(path.c_str());
     if (!t) throw Error(format("cannot read aterm from `%1%'") % path);
     return t;
 }
 
 
-FSId writeTerm(ATerm t, const string & suffix, string * p)
+FSId writeTerm(ATerm t, const string & suffix)
 {
     FSId id = hashTerm(t);
 
@@ -182,28 +40,14 @@ FSId writeTerm(ATerm t, const string & suffix, string * p)
     if (!ATwriteToNamedTextFile(t, path.c_str()))
         throw Error(format("cannot write aterm %1%") % path);
 
+//     debug(format("written term %1% = %2%") % (string) id %
+//         printTerm(t));
+
     registerPath(path, id);
-    if (p) *p = path;
-
     return id;
 }
 
 
-void registerSuccessor(const FSId & id1, const FSId & id2)
-{
-    setDB(nixDB, dbSuccessors, id1, id2);
-}
-
-
-static FSId storeSuccessor(const FSId & id1, FState sc,
-    string * p)
-{
-    FSId id2 = writeTerm(sc, "-s-" + (string) id1, p);
-    registerSuccessor(id1, id2);
-    return id2;
-}
-
-
 static void parseIds(ATermList ids, FSIds & out)
 {
     while (!ATisEmpty(ids)) {
@@ -217,251 +61,9 @@ static void parseIds(ATermList ids, FSIds & out)
 }
 
 
-static void checkSlice(const Slice & slice);
-
-
-/* Parse a slice. */
-static Slice parseSlice(FState fs)
-{
-    Slice slice;
-    ATermList roots, elems;
-    
-    if (!ATmatch(fs, "Slice([<list>], [<list>])", &roots, &elems))
-        throw badTerm("not a slice", fs);
-
-    parseIds(roots, slice.roots);
-
-    while (!ATisEmpty(elems)) {
-        char * s1, * s2;
-        ATermList refs;
-        ATerm t = ATgetFirst(elems);
-        if (!ATmatch(t, "(<str>, <str>, [<list>])", &s1, &s2, &refs))
-            throw badTerm("not a slice element", t);
-        SliceElem elem;
-        elem.path = s1;
-        elem.id = parseHash(s2);
-        parseIds(refs, elem.refs);
-        slice.elems.push_back(elem);
-        elems = ATgetNext(elems);
-    }
-
-    checkSlice(slice);
-
-    return slice;
-}
-
-
-static ATermList unparseIds(const FSIds & ids)
-{
-    ATermList l = ATempty;
-    for (FSIds::const_iterator i = ids.begin();
-         i != ids.end(); i++)
-        l = ATinsert(l,
-            ATmake("<str>", ((string) *i).c_str()));
-    return ATreverse(l);
-}
-
-
-static FState unparseSlice(const Slice & slice)
-{
-    ATermList roots = unparseIds(slice.roots);
-    
-    ATermList elems = ATempty;
-    for (SliceElems::const_iterator i = slice.elems.begin();
-         i != slice.elems.end(); i++)
-        elems = ATinsert(elems,
-            ATmake("(<str>, <str>, <term>)",
-                i->path.c_str(),
-                ((string) i->id).c_str(),
-                unparseIds(i->refs)));
-
-    return ATmake("Slice(<term>, <term>)", roots, elems);
-}
-
-
 typedef set<FSId> FSIdSet;
 
 
-static Slice normaliseFState2(FSId id, StringSet & usedPaths)
-{
-    debug(format("normalising fstate"));
-    Nest nest(true);
-
-    /* Try to substitute $id$ by any known successors in order to
-       speed up the rewrite process. */
-    string idSucc;
-    while (queryDB(nixDB, dbSuccessors, id, idSucc)) {
-        debug(format("successor %1% -> %2%") % (string) id % idSucc);
-        id = parseHash(idSucc);
-    }
-
-    /* Get the fstate expression. */
-    string fsPath;
-    FState fs = termFromId(id, &fsPath);
-
-    /* Already in normal form (i.e., a slice)? */
-    if (ATgetType(fs) == AT_APPL && 
-        (string) ATgetName(ATgetAFun(fs)) == "Slice")
-    {
-        usedPaths.insert(fsPath);
-        return parseSlice(fs);
-    }
-    
-    /* Then we it's a Derive node. */
-    ATermList outs, ins, bnds;
-    char * builder;
-    char * platform;
-    if (!ATmatch(fs, "Derive([<list>], [<list>], <str>, <str>, [<list>])",
-            &outs, &ins, &builder, &platform, &bnds))
-        throw badTerm("not a derive", fs);
-
-    /* Right platform? */
-    checkPlatform(platform);
-        
-    /* Realise inputs (and remember all input paths). */
-    FSIds inIds;
-    parseIds(ins, inIds);
-
-    typedef map<string, SliceElem> ElemMap;
-
-    ElemMap inMap;
-
-    for (FSIds::iterator i = inIds.begin(); i != inIds.end(); i++) {
-        Slice slice = normaliseFState(*i);
-        realiseSlice(slice);
-
-        for (SliceElems::iterator j = slice.elems.begin();
-             j != slice.elems.end(); j++)
-            inMap[j->path] = *j;
-    }
-
-    Strings inPaths;
-    for (ElemMap::iterator i = inMap.begin(); i != inMap.end(); i++)
-        inPaths.push_back(i->second.path);
-
-    /* Build the environment. */
-    Environment env;
-    while (!ATisEmpty(bnds)) {
-        char * s1, * s2;
-        ATerm bnd = ATgetFirst(bnds);
-        if (!ATmatch(bnd, "(<str>, <str>)", &s1, &s2))
-            throw badTerm("tuple of strings expected", bnd);
-        env[s1] = s2;
-        bnds = ATgetNext(bnds);
-    }
-
-    /* Parse the outputs. */
-    typedef map<string, FSId> OutPaths;
-    OutPaths outPaths;
-    while (!ATisEmpty(outs)) {
-        ATerm t = ATgetFirst(outs);
-        char * s1, * s2;
-        if (!ATmatch(t, "(<str>, <str>)", &s1, &s2))
-            throw badTerm("string expected", t);
-        outPaths[s1] = parseHash(s2);
-        inPaths.push_back(s1);
-        outs = ATgetNext(outs);
-    }
-
-    /* We can skip running the builder if we can expand all output
-       paths from their ids. */
-    bool fastBuild = false;
-#if 0
-    for (OutPaths::iterator i = outPaths.begin(); 
-         i != outPaths.end(); i++)
-    {
-        try {
-            expandId(i->second, i->first);
-        } catch (...) {
-            fastBuild = false;
-            break;
-        }
-    }
-#endif
-
-    if (!fastBuild) {
-
-        /* Check that none of the outputs exist. */
-        for (OutPaths::iterator i = outPaths.begin(); 
-             i != outPaths.end(); i++)
-            if (pathExists(i->first))
-                throw Error(format("path `%1%' exists") % i->first);
-
-        /* Run the builder. */
-        runProgram(builder, env);
-        
-    } else
-        debug(format("skipping build"));
-
-    Slice slice;
-
-    /* Check whether the output paths were created, and register each
-       one. */
-    FSIdSet used;
-    for (OutPaths::iterator i = outPaths.begin(); 
-         i != outPaths.end(); i++)
-    {
-        string path = i->first;
-        if (!pathExists(path))
-            throw Error(format("path `%1%' does not exist") % path);
-        registerPath(path, i->second);
-        slice.roots.push_back(i->second);
-
-        Strings refs = filterReferences(path, inPaths);
-
-        SliceElem elem;
-        elem.path = path;
-        elem.id = i->second;
-
-        for (Strings::iterator j = refs.begin(); j != refs.end(); j++) {
-            ElemMap::iterator k;
-            OutPaths::iterator l;
-            if ((k = inMap.find(*j)) != inMap.end()) {
-                elem.refs.push_back(k->second.id);
-                used.insert(k->second.id);
-                for (FSIds::iterator m = k->second.refs.begin();
-                     m != k->second.refs.end(); m++)
-                    used.insert(*m);
-            } else if ((l = outPaths.find(*j)) != outPaths.end()) {
-                elem.refs.push_back(l->second);
-                used.insert(l->second);
-            } else 
-                throw Error(format("unknown referenced path `%1%'") % *j);
-        }
-
-        slice.elems.push_back(elem);
-    }
-
-    for (ElemMap::iterator i = inMap.begin();
-         i != inMap.end(); i++)
-    {
-        FSIdSet::iterator j = used.find(i->second.id);
-        if (j == used.end())
-            debug(format("NOT referenced: `%1%'") % i->second.path);
-        else {
-            debug(format("referenced: `%1%'") % i->second.path);
-            slice.elems.push_back(i->second);
-        }
-    }
-
-    FState nf = unparseSlice(slice);
-    debug(printTerm(nf));
-    storeSuccessor(id, nf, &fsPath);
-    usedPaths.insert(fsPath);
-
-    parseSlice(nf); /* check */
-
-    return slice;
-}
-
-
-Slice normaliseFState(FSId id)
-{
-    StringSet dummy;
-    return normaliseFState2(id, dummy);
-}
-
-
 static void checkSlice(const Slice & slice)
 {
     if (slice.elems.size() == 0)
@@ -486,97 +88,142 @@ static void checkSlice(const Slice & slice)
 }
 
 
-void realiseSlice(const Slice & slice)
+/* Parse a slice. */
+static bool parseSlice(ATerm t, Slice & slice)
 {
-    debug(format("realising slice"));
+    ATermList roots, elems;
-    Nest nest(true);
     
-    /* Perhaps all paths already contain the right id? */
+    if (!ATmatch(t, "Slice([<list>], [<list>])", &roots, &elems))
+        return false;
 
-    bool missing = false;
+    parseIds(roots, slice.roots);
-    for (SliceElems::const_iterator i = slice.elems.begin();
+
-         i != slice.elems.end(); i++)
+    while (!ATisEmpty(elems)) {
-    {
+        char * s1, * s2;
-        SliceElem elem = *i;
+        ATermList refs;
-        string id;
+        ATerm t = ATgetFirst(elems);
-        if (!queryDB(nixDB, dbPath2Id, elem.path, id)) {
+        if (!ATmatch(t, "(<str>, <str>, [<list>])", &s1, &s2, &refs))
-            if (pathExists(elem.path))
+            throw badTerm("not a slice element", t);
-                throw Error(format("path `%1%' obstructed") % elem.path);
+        SliceElem elem;
-            missing = true;
+        elem.path = s1;
-            break;
+        elem.id = parseHash(s2);
-        }
+        parseIds(refs, elem.refs);
-        if (parseHash(id) != elem.id)
+        slice.elems.push_back(elem);
-            throw Error(format("path `%1%' obstructed") % elem.path);
+        elems = ATgetNext(elems);
     }
 
-    if (!missing) {
+    checkSlice(slice);
-        debug(format("already installed"));
+    return true;
-        return;
-    }
-
-    /* For each element, expand its id at its path. */
-    for (SliceElems::const_iterator i = slice.elems.begin();
-         i != slice.elems.end(); i++)
-    {
-        SliceElem elem = *i;
-        expandId(elem.id, elem.path);
-    }
 }
 
 
-Strings fstatePaths(const FSId & id, bool normalise)
+static bool parseDerive(ATerm t, Derive & derive)
 {
-    Strings paths;
-
-    FState fs = termFromId(id);
-
     ATermList outs, ins, bnds;
     char * builder;
     char * platform;
 
-    if (normalise ||
+    if (!ATmatch(t, "Derive([<list>], [<list>], <str>, <str>, [<list>])",
-        (ATgetType(fs) == AT_APPL && 
+            &outs, &ins, &builder, &platform, &bnds))
-         (string) ATgetName(ATgetAFun(fs)) == "Slice"))
+        return false;
-    {
-        Slice slice;
-        if (normalise)
-            slice = normaliseFState(id);
-        else
-            slice = parseSlice(fs);
 
-        /* !!! fix complexity */
+    while (!ATisEmpty(outs)) {
-        for (FSIds::const_iterator i = slice.roots.begin();
+        char * s1, * s2;
-             i != slice.roots.end(); i++)
+        ATerm t = ATgetFirst(outs);
-            for (SliceElems::const_iterator j = slice.elems.begin();
+        if (!ATmatch(t, "(<str>, <str>)", &s1, &s2))
-                 j != slice.elems.end(); j++)
+            throw badTerm("not a derive output", t);
-                if (*i == j->id) paths.push_back(j->path);
+        derive.outputs.push_back(DeriveOutput(s1, parseHash(s2)));
+        outs = ATgetNext(outs);
     }
 
-    else if (ATmatch(fs, "Derive([<list>], [<list>], <str>, <str>, [<list>])",
+    parseIds(ins, derive.inputs);
-                 &outs, &ins, &builder, &platform, &bnds))
+
-    {
+    derive.builder = builder;
-        while (!ATisEmpty(outs)) {
+    derive.platform = platform;
-            ATerm t = ATgetFirst(outs);
+    
-            char * s1, * s2;
+    while (!ATisEmpty(bnds)) {
-            if (!ATmatch(t, "(<str>, <str>)", &s1, &s2))
+        char * s1, * s2;
-                throw badTerm("string expected", t);
+        ATerm bnd = ATgetFirst(bnds);
-            paths.push_back(s1);
+        if (!ATmatch(bnd, "(<str>, <str>)", &s1, &s2))
-            outs = ATgetNext(outs);
+            throw badTerm("tuple of strings expected", bnd);
-        }
+        derive.env.push_back(StringPair(s1, s2));
+        bnds = ATgetNext(bnds);
     }
 
-    else throw badTerm("in fstatePaths", fs);
+    return true;
-
-    return paths;
 }
 
 
-StringSet fstateRefs(const FSId & id)
+FState parseFState(ATerm t)
 {
-    StringSet paths;
+    FState fs;
-    Slice slice = normaliseFState2(id, paths);
+    if (parseSlice(t, fs.slice))
+        fs.type = FState::fsSlice;
+    else if (parseDerive(t, fs.derive))
+        fs.type = FState::fsDerive;
+    else throw badTerm("not an fstate-expression", t);
+    return fs;
+}
+
+
+static ATermList unparseIds(const FSIds & ids)
+{
+    ATermList l = ATempty;
+    for (FSIds::const_iterator i = ids.begin();
+         i != ids.end(); i++)
+        l = ATinsert(l,
+            ATmake("<str>", ((string) *i).c_str()));
+    return ATreverse(l);
+}
+
+
+static ATerm unparseSlice(const Slice & slice)
+{
+    ATermList roots = unparseIds(slice.roots);
+    
+    ATermList elems = ATempty;
     for (SliceElems::const_iterator i = slice.elems.begin();
          i != slice.elems.end(); i++)
-        paths.insert(i->path);
+        elems = ATinsert(elems,
-    return paths;
+            ATmake("(<str>, <str>, <term>)",
+                i->path.c_str(),
+                ((string) i->id).c_str(),
+                unparseIds(i->refs)));
+
+    return ATmake("Slice(<term>, <term>)", roots, elems);
+}
+
+
+static ATerm unparseDerive(const Derive & derive)
+{
+    ATermList outs = ATempty;
+    for (DeriveOutputs::const_iterator i = derive.outputs.begin();
+         i != derive.outputs.end(); i++)
+        outs = ATinsert(outs,
+            ATmake("(<str>, <str>)", 
+                i->first.c_str(), ((string) i->second).c_str()));
+    
+    ATermList env = ATempty;
+    for (StringPairs::const_iterator i = derive.env.begin();
+         i != derive.env.end(); i++)
+        env = ATinsert(env,
+            ATmake("(<str>, <str>)", 
+                i->first.c_str(), i->second.c_str()));
+
+    return ATmake("Derive(<term>, <term>, <str>, <str>, <term>)",
+        ATreverse(outs),
+        unparseIds(derive.inputs),
+        derive.builder.c_str(),
+        derive.platform.c_str(),
+        ATreverse(env));
+}
+
+
+ATerm unparseFState(const FState & fs)
+{
+    if (fs.type == FState::fsSlice)
+        return unparseSlice(fs.slice);
+    else if (fs.type == FState::fsDerive)
+        return unparseDerive(fs.derive);
+    else abort();
 }

src/fstate.hh

@@ -5,55 +5,13 @@ extern "C" {
 #include <aterm2.h>
 }
 
-#include "hash.hh"
 #include "store.hh"
 
-using namespace std;
 
-
+/* Abstract syntax of fstate-expressions. */
-/* \section{Abstract syntax of Nix file system state expressions}
-
-   A Nix file system state expression, or FState, describes a
-   (partial) state of the file system.
-
-     Slice : [Id] * [(Path, Id, [Id])] -> FState
-
-   (update)
-   Path(path, content, refs) specifies a file object (its full path
-   and contents), along with all file objects referenced by it (that
-   is, that it has pointers to).  We assume that all files are
-   self-referential.  This prevents us from having to deal with
-   cycles.
-
-     Derive : [(Path, Id)] * [FStateId] * Path * [(String, String)] -> FState
-
-   (update)
-   Derive(platform, builder, ins, outs, env) specifies the creation of
-   new file objects (in paths declared by `outs') by the execution of
-   a program `builder' on a platform `platform'.  This execution takes
-   place in a file system state given by `ins'.  `env' specifies a
-   mapping of strings to strings.
-
-   A FState expression is in {\em $f$-normal form} if all Derive nodes
-   have been reduced to File nodes.
-
-   DISCUSSION: the idea is that a Regular/Directory is interchangeable
-   with its CHash.  This would appear to break referential
-   transparency, e.g., Derive(..., ..., [...CHash(h)...], ...) can
-   only be reduced in a context were the Regular/Directory equivalent
-   of Hash(h) is known.  However, CHash should be viewed strictly as a
-   shorthand; that is, when we export an expression containing a
-   CHash, we should also export the file object referenced by that
-   CHash.
-
-*/
-
-typedef ATerm FState;
-typedef ATerm Content;
 
 typedef list<FSId> FSIds;
 
-
 struct SliceElem
 {
     string path;
@@ -69,6 +27,27 @@ struct Slice
     SliceElems elems;
 };
 
+typedef pair<string, FSId> DeriveOutput;
+typedef pair<string, string> StringPair;
+typedef list<DeriveOutput> DeriveOutputs;
+typedef list<StringPair> StringPairs;
+
+struct Derive
+{
+    DeriveOutputs outputs;
+    FSIds inputs;
+    string builder;
+    string platform;
+    StringPairs env;
+};
+
+struct FState
+{
+    enum { fsSlice, fsDerive } type;
+    Slice slice;
+    Derive derive;
+};
+
 
 /* Return a canonical textual representation of an expression. */
 string printTerm(ATerm t);
@@ -81,28 +60,16 @@ Error badTerm(const format & f, ATerm t);
 Hash hashTerm(ATerm t);
 
 /* Read an aterm from disk, given its id. */
-ATerm termFromId(const FSId & id, string * p = 0);
+ATerm termFromId(const FSId & id);
 
 /* Write an aterm to the Nix store directory, and return its hash. */
-FSId writeTerm(ATerm t, const string & suffix, string * p = 0);
+FSId writeTerm(ATerm t, const string & suffix);
 
-/* Register a successor. */
+/* Parse an fstate-expression. */
-void registerSuccessor(const FSId & id1, const FSId & id2);
+FState parseFState(ATerm t);
 
-
+/* Parse an fstate-expression. */
-/* Normalise an fstate-expression, that is, return an equivalent
+ATerm unparseFState(const FState & fs);
-   Slice. */
-Slice normaliseFState(FSId id);
-
-/* Realise a Slice in the file system. */
-void realiseSlice(const Slice & slice);
-
-/* Get the list of root (output) paths of the given
-   fstate-expression. */
-Strings fstatePaths(const FSId & id, bool normalise);
-
-/* Get the list of paths referenced by the given fstate-expression. */
-StringSet fstateRefs(const FSId & id);
 
 
 #endif /* !__FSTATE_H */

src/nix.cc

@@ -1,8 +1,7 @@
 #include <iostream>
 
 #include "globals.hh"
-#include "store.hh"
+#include "normalise.hh"
-#include "fstate.hh"
 #include "archive.hh"
 #include "shared.hh"
 

src/normalise.cc

@@ -0,0 +1,265 @@
+#include <map>
+
+#include "normalise.hh"
+#include "references.hh"
+#include "db.hh"
+#include "exec.hh"
+#include "globals.hh"
+
+
+void registerSuccessor(const FSId & id1, const FSId & id2)
+{
+    setDB(nixDB, dbSuccessors, id1, id2);
+}
+
+
+static FSId storeSuccessor(const FSId & id1, ATerm sc)
+{
+    FSId id2 = writeTerm(sc, "-s-" + (string) id1);
+    registerSuccessor(id1, id2);
+    return id2;
+}
+
+
+typedef set<FSId> FSIdSet;
+
+
+Slice normaliseFState(FSId id)
+{
+    debug(format("normalising fstate %1%") % (string) id);
+    Nest nest(true);
+
+    /* Try to substitute $id$ by any known successors in order to
+       speed up the rewrite process. */
+    string idSucc;
+    while (queryDB(nixDB, dbSuccessors, id, idSucc)) {
+        debug(format("successor %1% -> %2%") % (string) id % idSucc);
+        id = parseHash(idSucc);
+    }
+
+    /* Get the fstate expression. */
+    FState fs = parseFState(termFromId(id));
+
+    /* It this is a normal form (i.e., a slice) we are done. */
+    if (fs.type == FState::fsSlice) return fs.slice;
+    
+    /* Otherwise, it's a derivation. */
+
+    /* Right platform? */
+    if (fs.derive.platform != thisSystem)
+        throw Error(format("a `%1%' is required, but I am a `%2%'")
+            % fs.derive.platform % thisSystem);
+        
+    /* Realise inputs (and remember all input paths). */
+    typedef map<string, SliceElem> ElemMap;
+
+    ElemMap inMap;
+
+    for (FSIds::iterator i = fs.derive.inputs.begin();
+         i != fs.derive.inputs.end(); i++) {
+        Slice slice = normaliseFState(*i);
+        realiseSlice(slice);
+
+        for (SliceElems::iterator j = slice.elems.begin();
+             j != slice.elems.end(); j++)
+            inMap[j->path] = *j;
+    }
+
+    Strings inPaths;
+    for (ElemMap::iterator i = inMap.begin(); i != inMap.end(); i++)
+        inPaths.push_back(i->second.path);
+
+    /* Build the environment. */
+    Environment env;
+    for (StringPairs::iterator i = fs.derive.env.begin();
+         i != fs.derive.env.end(); i++)
+        env[i->first] = i->second;
+
+    /* Parse the outputs. */
+    typedef map<string, FSId> OutPaths;
+    OutPaths outPaths;
+    for (DeriveOutputs::iterator i = fs.derive.outputs.begin();
+         i != fs.derive.outputs.end(); i++)
+    {
+        debug(format("building %1% in %2%") % (string) i->second % i->first);
+        outPaths[i->first] = i->second;
+        inPaths.push_back(i->first);
+    }
+
+    /* We can skip running the builder if we can expand all output
+       paths from their ids. */
+    bool fastBuild = false;
+#if 0
+    for (OutPaths::iterator i = outPaths.begin(); 
+         i != outPaths.end(); i++)
+    {
+        try {
+            expandId(i->second, i->first);
+        } catch (...) {
+            fastBuild = false;
+            break;
+        }
+    }
+#endif
+
+    if (!fastBuild) {
+
+        /* Check that none of the outputs exist. */
+        for (OutPaths::iterator i = outPaths.begin(); 
+             i != outPaths.end(); i++)
+            if (pathExists(i->first))
+                throw Error(format("path `%1%' exists") % i->first);
+
+        /* Run the builder. */
+        debug(format("building..."));
+        runProgram(fs.derive.builder, env);
+        debug(format("build completed"));
+        
+    } else
+        debug(format("skipping build"));
+
+    /* Check whether the output paths were created, and register each
+       one. */
+    FSIdSet used;
+    for (OutPaths::iterator i = outPaths.begin(); 
+         i != outPaths.end(); i++)
+    {
+        string path = i->first;
+        if (!pathExists(path))
+            throw Error(format("path `%1%' does not exist") % path);
+        registerPath(path, i->second);
+        fs.slice.roots.push_back(i->second);
+
+        Strings refs = filterReferences(path, inPaths);
+
+        SliceElem elem;
+        elem.path = path;
+        elem.id = i->second;
+
+        for (Strings::iterator j = refs.begin(); j != refs.end(); j++) {
+            ElemMap::iterator k;
+            OutPaths::iterator l;
+            if ((k = inMap.find(*j)) != inMap.end()) {
+                elem.refs.push_back(k->second.id);
+                used.insert(k->second.id);
+                for (FSIds::iterator m = k->second.refs.begin();
+                     m != k->second.refs.end(); m++)
+                    used.insert(*m);
+            } else if ((l = outPaths.find(*j)) != outPaths.end()) {
+                elem.refs.push_back(l->second);
+                used.insert(l->second);
+            } else 
+                throw Error(format("unknown referenced path `%1%'") % *j);
+        }
+
+        fs.slice.elems.push_back(elem);
+    }
+
+    for (ElemMap::iterator i = inMap.begin();
+         i != inMap.end(); i++)
+    {
+        FSIdSet::iterator j = used.find(i->second.id);
+        if (j == used.end())
+            debug(format("NOT referenced: `%1%'") % i->second.path);
+        else {
+            debug(format("referenced: `%1%'") % i->second.path);
+            fs.slice.elems.push_back(i->second);
+        }
+    }
+
+    fs.type = FState::fsSlice;
+    ATerm nf = unparseFState(fs);
+    debug(format("normal form: %1%") % printTerm(nf));
+    storeSuccessor(id, nf);
+
+    return fs.slice;
+}
+
+
+void realiseSlice(const Slice & slice)
+{
+    debug(format("realising slice"));
+    Nest nest(true);
+
+    /* Perhaps all paths already contain the right id? */
+
+    bool missing = false;
+    for (SliceElems::const_iterator i = slice.elems.begin();
+         i != slice.elems.end(); i++)
+    {
+        SliceElem elem = *i;
+        string id;
+        if (!queryDB(nixDB, dbPath2Id, elem.path, id)) {
+            if (pathExists(elem.path))
+                throw Error(format("path `%1%' obstructed") % elem.path);
+            missing = true;
+            break;
+        }
+        if (parseHash(id) != elem.id)
+            throw Error(format("path `%1%' obstructed") % elem.path);
+    }
+
+    if (!missing) {
+        debug(format("already installed"));
+        return;
+    }
+
+    /* For each element, expand its id at its path. */
+    for (SliceElems::const_iterator i = slice.elems.begin();
+         i != slice.elems.end(); i++)
+    {
+        SliceElem elem = *i;
+        debug(format("expanding %1% in %2%") % (string) elem.id % elem.path);
+        expandId(elem.id, elem.path);
+    }
+}
+
+
+Strings fstatePaths(const FSId & id, bool normalise)
+{
+    Strings paths;
+
+    FState fs;
+
+    if (normalise) {
+        fs.slice = normaliseFState(id);
+        fs.type = FState::fsSlice;
+    } else
+        fs = parseFState(termFromId(id));
+
+    if (fs.type == FState::fsSlice) {
+        /* !!! fix complexity */
+        for (FSIds::const_iterator i = fs.slice.roots.begin();
+             i != fs.slice.roots.end(); i++)
+            for (SliceElems::const_iterator j = fs.slice.elems.begin();
+                 j != fs.slice.elems.end(); j++)
+                if (*i == j->id) paths.push_back(j->path);
+    }
+
+    else if (fs.type == FState::fsDerive) {
+        for (DeriveOutputs::iterator i = fs.derive.outputs.begin();
+             i != fs.derive.outputs.end(); i++)
+            paths.push_back(i->first);
+    }
+    
+    else abort();
+
+    return paths;
+}
+
+
+StringSet fstateRefs(const FSId & id)
+{
+    StringSet paths;
+    Slice slice = normaliseFState(id);
+    for (SliceElems::const_iterator i = slice.elems.begin();
+         i != slice.elems.end(); i++)
+        paths.insert(i->path);
+    return paths;
+}
+
+
+void findGenerators(const FSIds & ids)
+{
+    
+}

src/normalise.hh

@@ -0,0 +1,25 @@
+#ifndef __NORMALISE_H
+#define __NORMALISE_H
+
+#include "fstate.hh"
+
+
+/* Normalise an fstate-expression, that is, return an equivalent
+   Slice. */
+Slice normaliseFState(FSId id);
+
+/* Realise a Slice in the file system. */
+void realiseSlice(const Slice & slice);
+
+/* Get the list of root (output) paths of the given
+   fstate-expression. */
+Strings fstatePaths(const FSId & id, bool normalise);
+
+/* Get the list of paths referenced by the given fstate-expression. */
+StringSet fstateRefs(const FSId & id);
+
+/* Register a successor. */
+void registerSuccessor(const FSId & id1, const FSId & id2);
+
+
+#endif /* !__NORMALISE_H */

src/store.cc

@@ -7,7 +7,7 @@
 #include "globals.hh"
 #include "db.hh"
 #include "archive.hh"
-#include "fstate.hh"
+#include "normalise.hh"
 
 
 struct CopySink : DumpSink

src/test.cc

@@ -6,14 +6,14 @@
 #include "hash.hh"
 #include "archive.hh"
 #include "util.hh"
-#include "fstate.hh"
+#include "normalise.hh"
-#include "store.hh"
 #include "globals.hh"
 
 
 void realise(FSId id)
 {
-    cout << format("realising %1%\n") % (string) id;
+    debug(format("TEST: realising %1%") % (string) id);
+    Nest nest(true);
     Slice slice = normaliseFState(id);
     realiseSlice(slice);
 }
@@ -117,7 +117,7 @@ void runTests()
     string builder1fn;
     addToStore("./test-builder-1.sh", builder1fn, builder1id);
 
-    FState fs1 = ATmake(
+    ATerm fs1 = ATmake(
         "Slice([<str>], [(<str>, <str>, [])])",
         ((string) builder1id).c_str(),
         builder1fn.c_str(),
@@ -127,7 +127,7 @@ void runTests()
     realise(fs1id);
     realise(fs1id);
 
-    FState fs2 = ATmake(
+    ATerm fs2 = ATmake(
         "Slice([<str>], [(<str>, <str>, [])])",
         ((string) builder1id).c_str(),
         (builder1fn + "_bla").c_str(),
@@ -139,7 +139,7 @@ void runTests()
 
     string out1id = hashString("foo"); /* !!! bad */
     string out1fn = nixStore + "/" + (string) out1id + "-hello.txt";
-    FState fs3 = ATmake(
+    ATerm fs3 = ATmake(
         "Derive([(<str>, <str>)], [<str>], <str>, <str>, [(\"out\", <str>)])",
         out1fn.c_str(),
         ((string) out1id).c_str(),
@@ -158,7 +158,7 @@ void runTests()
     string builder4fn;
     addToStore("./test-builder-2.sh", builder4fn, builder4id);
 
-    FState fs4 = ATmake(
+    ATerm fs4 = ATmake(
         "Slice([<str>], [(<str>, <str>, [])])",
         ((string) builder4id).c_str(),
         builder4fn.c_str(),
@@ -169,7 +169,7 @@ void runTests()
 
     string out5id = hashString("bar"); /* !!! bad */
     string out5fn = nixStore + "/" + (string) out5id + "-hello2";
-    FState fs5 = ATmake(
+    ATerm fs5 = ATmake(
         "Derive([(<str>, <str>)], [<str>], <str>, <str>, [(\"out\", <str>), (\"builder\", <str>)])",
         out5fn.c_str(),
         ((string) out5id).c_str(),
@@ -183,27 +183,6 @@ void runTests()
 
     realise(fs5id);
     realise(fs5id);
-
-
-#if 0
-    FState fs2 = ATmake(
-        "Path(<str>, Hash(<str>), [])", 
-        (builder1fn + "_bla").c_str(),
-        ((string) builder1h).c_str());
-    realise(fs2);
-    realise(fs2);
-
-    string out1fn = nixStore + "/hello.txt";
-    FState fs3 = ATmake(
-        "Derive(<str>, <str>, [<term>], <str>, [(\"out\", <str>)])",
-        thisSystem.c_str(),
-        builder1fn.c_str(),
-        fs1,
-        out1fn.c_str(),
-        out1fn.c_str());
-    realise(fs3);
-#endif
-
 }
 
 

src/util.cc

@@ -150,7 +150,7 @@ void msg(const format & f)
 {
     string spaces;
     for (int i = 0; i < nestingLevel; i++)
-        spaces += "  ";
+        spaces += "|   ";
     cerr << format("%1%%2%\n") % spaces % f.str();
 }
 

substitute.mk

@@ -0,0 +1,8 @@
+%: %.in Makefile
+	sed \
+	 -e s^@prefix\@^$(prefix)^g \
+	 -e s^@bindir\@^$(bindir)^g \
+	 -e s^@sysconfdir\@^$(sysconfdir)^g \
+	 -e s^@localstatedir\@^$(localstatedir)^g \
+	 < $< > $@ || rm $@
+	chmod +x $@