boost*: use packaged b2 instead it of building in configurePhase

If we build two things in one derivation, it becomes more complicated
and its build time is extended. Therefore we should avoid this if
possible. There's a good opportunity for this with boost: We have
boost-build packaged already. This has the additional benefit that
we can get rid of $CC_FOR_BUILD entirely in boost, meaning we don't
need to rely on (as many) hacks to make boost understand our way of
cross compiling.

Unfortunately boost-build is not backwards compatible, so we need to
build a specific boost-build for each boost derivation (the number
could probably be reduced, but I'm not interested in testing a lot
of boost builds at the moment).

Additionally we fix a few cross compilation problems:

- boost couldn't cope with different types of compilers for native
  and cross (as happens if useLLVM is true). Since we only use one
  of them per derivation, this is no longer an issue.

- boost didn't find the cross ar and ranlib for compilation (since
  it doesn't check $AR or $RANLIB). Instead it used plain ar and
  ranlib form $PATH which were the native ones before. This is now
  fixed by setting these tools explicitly in user-config.jam (and
  no longer providing the native tools).

With these changes, pkgsLLVM.boost builds.

On darwin, instead of patching the clang-darwin.jam definition, we
instead supply -rpath $out/lib via <linkflags> which causes the
correct directory to be added to the libraries' rpaths, so that
they find each other.

pkgs/development/libraries/boost/default.nix

@@ -0,0 +1,47 @@
+{ lib
+, callPackage
+, boost-build
+, fetchurl
+}:
+
+let
+  # for boost 1.55 we need to use 1.56's b2
+  # since 1.55's build system is not working
+  # with our derivation
+  useBoost156 = rec {
+    version = "1.56.0";
+    src = fetchurl {
+      url = "mirror://sourceforge/boost/boost_${lib.replaceStrings ["."] ["_"] version}.tar.bz2";
+      sha256 = "07gz62nj767qzwqm3xjh11znpyph8gcii0cqhnx7wvismyn34iqk";
+    };
+  };
+
+  makeBoost = file:
+    lib.fix (self:
+      callPackage file {
+        boost-build = boost-build.override {
+          # useBoost allows us passing in src and version from
+          # the derivation we are building to get a matching b2 version.
+          useBoost =
+            if lib.versionAtLeast self.version "1.56"
+            then self
+            else useBoost156; # see above
+        };
+      }
+    );
+in {
+  boost155 = makeBoost ./1.55.nix;
+  boost159 = makeBoost ./1.59.nix;
+  boost160 = makeBoost ./1.60.nix;
+  boost165 = makeBoost ./1.65.nix;
+  boost166 = makeBoost ./1.66.nix;
+  boost167 = makeBoost ./1.67.nix;
+  boost168 = makeBoost ./1.68.nix;
+  boost169 = makeBoost ./1.69.nix;
+  boost170 = makeBoost ./1.70.nix;
+  boost171 = makeBoost ./1.71.nix;
+  boost172 = makeBoost ./1.72.nix;
+  boost173 = makeBoost ./1.73.nix;
+  boost174 = makeBoost ./1.74.nix;
+  boost175 = makeBoost ./1.75.nix;
+}

pkgs/development/libraries/boost/generic.nix

@@ -1,7 +1,7 @@
 { lib, stdenv, icu, expat, zlib, bzip2, python ? null, fixDarwinDylibNames, libiconv
+, boost-build
 , fetchpatch
 , which
-, buildPackages
 , toolset ? /**/ if stdenv.cc.isClang  then "clang"
             else null
 , enableRelease ? true
@@ -67,6 +67,8 @@ let
     else
       "$NIX_BUILD_CORES";
 
+  needUserConfig = stdenv.hostPlatform != stdenv.buildPlatform || useMpi || stdenv.isDarwin;
+
   b2Args = concatStringsSep " " ([
     "--includedir=$dev/include"
     "--libdir=$out/lib"
@@ -95,7 +97,7 @@ let
     ++ optional (variant == "release") "debug-symbols=off"
     ++ optional (toolset != null) "toolset=${toolset}"
     ++ optional (!enablePython) "--without-python"
-    ++ optional (useMpi || stdenv.hostPlatform != stdenv.buildPlatform) "--user-config=user-config.jam"
+    ++ optional needUserConfig "--user-config=user-config.jam"
     ++ optionals (stdenv.hostPlatform.libc == "msvcrt") [
     "threadapi=win32"
   ] ++ extraB2Args
@@ -137,22 +139,39 @@ stdenv.mkDerivation {
     maintainers = with maintainers; [ peti ];
   };
 
-  preConfigure = ''
-    if test -f tools/build/src/tools/clang-darwin.jam ; then
-        substituteInPlace tools/build/src/tools/clang-darwin.jam \
-          --replace '@rpath/$(<[1]:D=)' "$out/lib/\$(<[1]:D=)";
-    fi;
-  '' + optionalString useMpi ''
+  preConfigure = optionalString useMpi ''
     cat << EOF >> user-config.jam
     using mpi : ${mpi}/bin/mpiCC ;
     EOF
-  '' + optionalString (stdenv.hostPlatform != stdenv.buildPlatform) ''
+  ''
+  # On darwin we need to add the `$out/lib` to the libraries' rpath explicitly,
+  # otherwise the dynamic linker is unable to resolve the reference to @rpath
+  # when the boost libraries want to load each other at runtime.
+  + optionalString (stdenv.isDarwin && enableShared) ''
     cat << EOF >> user-config.jam
-    using gcc : cross : ${stdenv.cc.targetPrefix}c++ ;
+    using clang-darwin : : ${stdenv.cc.targetPrefix}c++
+      : <linkflags>"-rpath $out/lib/"
+      ;
+    EOF
+  ''
+  # b2 has trouble finding the correct compiler and tools for cross compilation
+  # since it apparently ignores $CC, $AR etc. Thus we need to set everything
+  # in user-config.jam. To keep things simple we just set everything in an
+  # uniform way for clang and gcc (which works thanks to our cc-wrapper).
+  # We pass toolset later which will make b2 invoke everything in the right
+  # way -- the other toolset in user-config.jam will be ignored.
+  + optionalString (stdenv.hostPlatform != stdenv.buildPlatform) ''
+    cat << EOF >> user-config.jam
+    using gcc : cross : ${stdenv.cc.targetPrefix}c++
+      : <archiver>$AR
+        <ranlib>$RANLIB
+      ;
+
+    using clang : cross : ${stdenv.cc.targetPrefix}c++
+      : <archiver>$AR
+        <ranlib>$RANLIB
+      ;
     EOF
-    # Build b2 with buildPlatform CC/CXX.
-    sed '2i export CC=$CC_FOR_BUILD; export CXX=$CXX_FOR_BUILD' \
-      -i ./tools/build/src/engine/build.sh
   '';
 
   NIX_CFLAGS_LINK = lib.optionalString stdenv.isDarwin
@@ -160,9 +179,8 @@ stdenv.mkDerivation {
 
   enableParallelBuilding = true;
 
-  nativeBuildInputs = [ which ]
+  nativeBuildInputs = [ which boost-build ]
     ++ optional stdenv.hostPlatform.isDarwin fixDarwinDylibNames;
-  depsBuildBuild = [ buildPackages.stdenv.cc ];
   buildInputs = [ expat zlib bzip2 libiconv ]
     ++ optional (stdenv.hostPlatform == stdenv.buildPlatform) icu
     ++ optional enablePython python
@@ -173,13 +191,13 @@ stdenv.mkDerivation {
   configureFlags = [
     "--includedir=$(dev)/include"
     "--libdir=$(out)/lib"
+    "--with-bjam=b2" # prevent bootstrapping b2 in configurePhase
   ] ++ optional enablePython "--with-python=${python.interpreter}"
-    ++ [ (if stdenv.hostPlatform == stdenv.buildPlatform then "--with-icu=${icu.dev}" else "--without-icu") ]
-    ++ optional (toolset != null) "--with-toolset=${toolset}";
+    ++ [ (if stdenv.hostPlatform == stdenv.buildPlatform then "--with-icu=${icu.dev}" else "--without-icu") ];
 
   buildPhase = ''
     runHook preBuild
-    ./b2 ${b2Args}
+    b2 ${b2Args}
     runHook postBuild
   '';
 
@@ -191,7 +209,7 @@ stdenv.mkDerivation {
     cp -a tools/boostbook/{xsl,dtd} $dev/share/boostbook/
 
     # Let boost install everything else
-    ./b2 ${b2Args} install
+    b2 ${b2Args} install
 
     runHook postInstall
   '';

pkgs/top-level/all-packages.nix

@@ -14815,22 +14815,25 @@ in
 
   boolstuff = callPackage ../development/libraries/boolstuff { };
 
-  boost155 = callPackage ../development/libraries/boost/1.55.nix { };
-  boost159 = callPackage ../development/libraries/boost/1.59.nix { };
+  inherit (callPackage ../development/libraries/boost { inherit (buildPackages) boost-build; })
+    boost155
+    boost159
+    boost160
+    boost165
+    boost166
+    boost167
+    boost168
+    boost169
+    boost170
+    boost171
+    boost172
+    boost173
+    boost174
+    boost175
+  ;
+
   boost15x = boost159;
-  boost160 = callPackage ../development/libraries/boost/1.60.nix { };
-  boost165 = callPackage ../development/libraries/boost/1.65.nix { };
-  boost166 = callPackage ../development/libraries/boost/1.66.nix { };
-  boost167 = callPackage ../development/libraries/boost/1.67.nix { };
-  boost168 = callPackage ../development/libraries/boost/1.68.nix { };
-  boost169 = callPackage ../development/libraries/boost/1.69.nix { };
   boost16x = boost169;
-  boost170 = callPackage ../development/libraries/boost/1.70.nix { };
-  boost171 = callPackage ../development/libraries/boost/1.71.nix { };
-  boost172 = callPackage ../development/libraries/boost/1.72.nix { };
-  boost173 = callPackage ../development/libraries/boost/1.73.nix { };
-  boost174 = callPackage ../development/libraries/boost/1.74.nix { };
-  boost175 = callPackage ../development/libraries/boost/1.75.nix { };
   boost17x = boost175;
   boost = boost16x;