Merge pull request #301416 from ConnorBaker/feat/cudaPackages-lessons-learned-from-cross-compilation-attempt

cuda-modules: apply lessons learned from cross-compilation attempts

pkgs/development/cuda-modules/backend-stdenv.nix

@@ -1,11 +1,9 @@
 {
+  cudaVersion,
   lib,
   nvccCompatibilities,
-  cudaVersion,
   pkgs,
-  overrideCC,
   stdenv,
-  wrapCCWith,
   stdenvAdapters,
 }:
 

pkgs/development/cuda-modules/cuda-library-samples/extension.nix

@@ -1,5 +1,7 @@
-{ hostPlatform, lib }:
+{ lib, stdenv }:
 let
+  inherit (stdenv) hostPlatform;
+
   # Samples are built around the CUDA Toolkit, which is not available for
   # aarch64. Check for both CUDA version and platform.
   platformIsSupported = hostPlatform.isx86_64 && hostPlatform.isLinux;

pkgs/development/cuda-modules/cuda-library-samples/generic.nix

@@ -76,7 +76,7 @@ in
       # CUTENSOR_ROOT is double escaped
       postPatch = ''
         substituteInPlace CMakeLists.txt \
-          --replace "\''${CUTENSOR_ROOT}/include" "${cutensor.dev}/include"
+          --replace-fail "\''${CUTENSOR_ROOT}/include" "${cutensor.dev}/include"
       '';
 
       CUTENSOR_ROOT = cutensor;

pkgs/development/cuda-modules/cuda-samples/extension.nix

@@ -1,7 +1,7 @@
 {
   cudaVersion,
-  hostPlatform,
   lib,
+  stdenv,
 }:
 let
   cudaVersionToHash = {
@@ -23,6 +23,8 @@ let
     "12.3" = "sha256-fjVp0G6uRCWxsfe+gOwWTN+esZfk0O5uxS623u0REAk=";
   };
 
+  inherit (stdenv) hostPlatform;
+
   # Samples are built around the CUDA Toolkit, which is not available for
   # aarch64. Check for both CUDA version and platform.
   cudaVersionIsSupported = cudaVersionToHash ? ${cudaVersion};

pkgs/development/cuda-modules/cuda-samples/generic.nix

@@ -11,6 +11,7 @@
   hash,
   lib,
   pkg-config,
+  stdenv,
 }:
 let
   inherit (lib) lists strings;
@@ -63,7 +64,7 @@ backendStdenv.mkDerivation (finalAttrs: {
   installPhase = ''
     runHook preInstall
 
-    install -Dm755 -t $out/bin bin/${backendStdenv.hostPlatform.parsed.cpu.name}/${backendStdenv.hostPlatform.parsed.kernel.name}/release/*
+    install -Dm755 -t $out/bin bin/${stdenv.hostPlatform.parsed.cpu.name}/${stdenv.hostPlatform.parsed.kernel.name}/release/*
 
     runHook postInstall
   '';

pkgs/development/cuda-modules/cuda/overrides.nix

@@ -1,122 +1,178 @@
-{
-  cudaVersion,
-  lib,
-  addDriverRunpath,
-}:
 let
-  inherit (lib) attrsets lists strings;
+  filterAndCreateOverrides =
-  # cudaVersionOlder : Version -> Boolean
+    createOverrideAttrs: final: prev:
-  cudaVersionOlder = strings.versionOlder cudaVersion;
+    let
-  # cudaVersionAtLeast : Version -> Boolean
+      # It is imperative that we use `final.callPackage` to perform overrides,
-  cudaVersionAtLeast = strings.versionAtLeast cudaVersion;
+      # so the final package set is available to the override functions.
+      inherit (final) callPackage;
 
-  addBuildInputs =
+      # NOTE(@connorbaker): We MUST use `lib` from `prev` because the attribute
-    drv: buildInputs:
+      # names CAN NOT depend on `final`.
-    drv.overrideAttrs (prevAttrs: {
+      inherit (prev.lib.attrsets) filterAttrs mapAttrs;
-      buildInputs = prevAttrs.buildInputs ++ buildInputs;
+      inherit (prev.lib.trivial) pipe;
-    });
+
-in
+      # NOTE: Filter out attributes that are not present in the previous version of
-# NOTE: Filter out attributes that are not present in the previous version of
+      # the package set. This is necessary to prevent the appearance of attributes
-# the package set. This is necessary to prevent the appearance of attributes
+      # like `cuda_nvcc` in `cudaPackages_10_0, which predates redistributables.
-# like `cuda_nvcc` in `cudaPackages_10_0, which predates redistributables.
+      filterOutNewAttrs = filterAttrs (name: _: prev ? ${name});
-final: prev:
+
-attrsets.filterAttrs (attr: _: (builtins.hasAttr attr prev)) {
+      # Apply callPackage to each attribute value, yielding a value to be passed
-  libcufile = prev.libcufile.overrideAttrs (prevAttrs: {
+      # to overrideAttrs.
-    buildInputs = prevAttrs.buildInputs ++ [
+      callPackageThenOverrideAttrs = mapAttrs (
-      final.libcublas.lib
+        name: value: prev.${name}.overrideAttrs (callPackage value { })
-      final.pkgs.numactl
+      );
-      final.pkgs.rdma-core
+    in
+    pipe createOverrideAttrs [
+      filterOutNewAttrs
+      callPackageThenOverrideAttrs
     ];
-    # Before 11.7 libcufile depends on itself for some reason.
+in
-    autoPatchelfIgnoreMissingDeps =
+# Each attribute name is the name of an existing package in the previous version
-      prevAttrs.autoPatchelfIgnoreMissingDeps
+# of the package set.
-      ++ lists.optionals (cudaVersionOlder "11.7") [ "libcufile.so.0" ];
+# The value is a function (to be provided to callPackage), which yields a value
-  });
+# to be provided to overrideAttrs. This allows us to override the attributes of
+# a package without losing access to the fixed point of the package set --
+# especially useful given that some packages may depend on each other!
+filterAndCreateOverrides {
+  libcufile =
+    {
+      cudaOlder,
+      lib,
+      libcublas,
+      numactl,
+      rdma-core,
+    }:
+    prevAttrs: {
+      buildInputs = prevAttrs.buildInputs ++ [
+        libcublas.lib
+        numactl
+        rdma-core
+      ];
+      # Before 11.7 libcufile depends on itself for some reason.
+      autoPatchelfIgnoreMissingDeps =
+        prevAttrs.autoPatchelfIgnoreMissingDeps
+        ++ lib.lists.optionals (cudaOlder "11.7") [ "libcufile.so.0" ];
+    };
 
-  libcusolver = addBuildInputs prev.libcusolver (
+  libcusolver =
-    # Always depends on this
+    {
-    [ final.libcublas.lib ]
+      cudaAtLeast,
-    # Dependency from 12.0 and on
+      lib,
-    ++ lists.optionals (cudaVersionAtLeast "12.0") [ final.libnvjitlink.lib ]
+      libcublas,
-    # Dependency from 12.1 and on
+      libcusparse ? null,
-    ++ lists.optionals (cudaVersionAtLeast "12.1") [ final.libcusparse.lib ]
+      libnvjitlink ? null,
-  );
+    }:
+    prevAttrs: {
+      buildInputs =
+        prevAttrs.buildInputs
+        # Always depends on this
+        ++ [ libcublas.lib ]
+        # Dependency from 12.0 and on
+        ++ lib.lists.optionals (cudaAtLeast "12.0") [ libnvjitlink.lib ]
+        # Dependency from 12.1 and on
+        ++ lib.lists.optionals (cudaAtLeast "12.1") [ libcusparse.lib ];
 
-  libcusparse = addBuildInputs prev.libcusparse (
+      brokenConditions = prevAttrs.brokenConditions // {
-    lists.optionals (cudaVersionAtLeast "12.0") [ final.libnvjitlink.lib ]
+        "libnvjitlink missing (CUDA >= 12.0)" =
-  );
+          !(cudaAtLeast "12.0" -> (libnvjitlink != null && libnvjitlink.lib != null));
+        "libcusparse missing (CUDA >= 12.1)" =
+          !(cudaAtLeast "12.1" -> (libcusparse != null && libcusparse.lib != null));
+      };
+    };
 
-  cuda_cudart = prev.cuda_cudart.overrideAttrs (prevAttrs: {
+  libcusparse =
-    # Remove once cuda-find-redist-features has a special case for libcuda
+    {
-    outputs =
+      cudaAtLeast,
-      prevAttrs.outputs
+      lib,
-      ++ lists.optionals (!(builtins.elem "stubs" prevAttrs.outputs)) [ "stubs" ];
+      libnvjitlink ? null,
+    }:
+    prevAttrs: {
+      buildInputs =
+        prevAttrs.buildInputs
+        # Dependency from 12.0 and on
+        ++ lib.lists.optionals (cudaAtLeast "12.0") [ libnvjitlink.lib ];
 
-    allowFHSReferences = false;
+      brokenConditions = prevAttrs.brokenConditions // {
+        "libnvjitlink missing (CUDA >= 12.0)" =
+          !(cudaAtLeast "12.0" -> (libnvjitlink != null && libnvjitlink.lib != null));
+      };
+    };
 
-    # The libcuda stub's pkg-config doesn't follow the general pattern:
+  # TODO(@connorbaker): cuda_cudart.dev depends on crt/host_config.h, which is from
-    postPatch =
+  # cuda_nvcc.dev. It would be nice to be able to encode that.
-      prevAttrs.postPatch or ""
+  cuda_cudart =
-      + ''
+    { addDriverRunpath, lib }:
-        while IFS= read -r -d $'\0' path ; do
+    prevAttrs: {
-          sed -i \
+      # Remove once cuda-find-redist-features has a special case for libcuda
-            -e "s|^libdir\s*=.*/lib\$|libdir=''${!outputLib}/lib/stubs|" \
+      outputs =
-            -e "s|^Libs\s*:\(.*\)\$|Libs: \1 -Wl,-rpath,${addDriverRunpath.driverLink}/lib|" \
+        prevAttrs.outputs
-            "$path"
+        ++ lib.lists.optionals (!(builtins.elem "stubs" prevAttrs.outputs)) [ "stubs" ];
-        done < <(find -iname 'cuda-*.pc' -print0)
+
-      ''
+      allowFHSReferences = false;
-      + ''
+
+      # The libcuda stub's pkg-config doesn't follow the general pattern:
+      postPatch =
+        prevAttrs.postPatch or ""
+        + ''
+          while IFS= read -r -d $'\0' path; do
+            sed -i \
+              -e "s|^libdir\s*=.*/lib\$|libdir=''${!outputLib}/lib/stubs|" \
+              -e "s|^Libs\s*:\(.*\)\$|Libs: \1 -Wl,-rpath,${addDriverRunpath.driverLink}/lib|" \
+              "$path"
+          done < <(find -iname 'cuda-*.pc' -print0)
+        ''
         # Namelink may not be enough, add a soname.
         # Cf. https://gitlab.kitware.com/cmake/cmake/-/issues/25536
-        if [[ -f lib/stubs/libcuda.so && ! -f lib/stubs/libcuda.so.1 ]] ; then
+        + ''
-          ln -s libcuda.so lib/stubs/libcuda.so.1
+          if [[ -f lib/stubs/libcuda.so && ! -f lib/stubs/libcuda.so.1 ]]; then
-        fi
+            ln -s libcuda.so lib/stubs/libcuda.so.1
-      '';
+          fi
+        '';
 
-    postFixup =
+      postFixup =
-      prevAttrs.postFixup or ""
+        prevAttrs.postFixup or ""
-      + ''
+        + ''
-        moveToOutput lib/stubs "$stubs"
+          moveToOutput lib/stubs "$stubs"
-        ln -s "$stubs"/lib/stubs/* "$stubs"/lib/
+          ln -s "$stubs"/lib/stubs/* "$stubs"/lib/
-        ln -s "$stubs"/lib/stubs "''${!outputLib}/lib/stubs"
+          ln -s "$stubs"/lib/stubs "''${!outputLib}/lib/stubs"
-      '';
+        '';
-  });
-
-  cuda_compat = prev.cuda_compat.overrideAttrs (prevAttrs: {
-    autoPatchelfIgnoreMissingDeps = prevAttrs.autoPatchelfIgnoreMissingDeps ++ [
-      "libnvrm_gpu.so"
-      "libnvrm_mem.so"
-      "libnvdla_runtime.so"
-    ];
-    # `cuda_compat` only works on aarch64-linux, and only when building for Jetson devices.
-    badPlatformsConditions = prevAttrs.badPlatformsConditions // {
-      "Trying to use cuda_compat on aarch64-linux targeting non-Jetson devices" =
-        !final.flags.isJetsonBuild;
     };
-  });
 
-  cuda_gdb = addBuildInputs prev.cuda_gdb (
+  cuda_compat =
-    # x86_64 only needs gmp from 12.0 and on
+    { flags, lib }:
-    lists.optionals (cudaVersionAtLeast "12.0") [ final.pkgs.gmp ]
+    prevAttrs: {
-  );
+      autoPatchelfIgnoreMissingDeps = prevAttrs.autoPatchelfIgnoreMissingDeps ++ [
-
+        "libnvrm_gpu.so"
-  cuda_nvcc = prev.cuda_nvcc.overrideAttrs (
+        "libnvrm_mem.so"
-    oldAttrs:
+        "libnvdla_runtime.so"
-    let
+      ];
-      # This replicates the logic in stdenvAdapters.useLibsFrom, except we use
+      # `cuda_compat` only works on aarch64-linux, and only when building for Jetson devices.
-      # gcc from pkgsHostTarget and not from buildPackages.
+      badPlatformsConditions = prevAttrs.badPlatformsConditions // {
-      ccForLibs-wrapper = final.pkgs.stdenv.cc;
+        "Trying to use cuda_compat on aarch64-linux targeting non-Jetson devices" = !flags.isJetsonBuild;
-      gccMajorVersion = final.nvccCompatibilities.${cudaVersion}.gccMaxMajorVersion;
-      cc = final.pkgs.wrapCCWith {
-        cc = final.pkgs."gcc${gccMajorVersion}".cc;
-        useCcForLibs = true;
-        gccForLibs = ccForLibs-wrapper.cc;
       };
-    in
+    };
+
+  cuda_gdb =
     {
+      cudaAtLeast,
+      gmp,
+      lib,
+    }:
+    prevAttrs: {
+      buildInputs =
+        prevAttrs.buildInputs
+        # x86_64 only needs gmp from 12.0 and on
+        ++ lib.lists.optionals (cudaAtLeast "12.0") [ gmp ];
+    };
 
-      outputs = oldAttrs.outputs ++ lists.optionals (!(builtins.elem "lib" oldAttrs.outputs)) [ "lib" ];
+  cuda_nvcc =
-
+    {
+      backendStdenv,
+      cuda_cudart,
+      lib,
+      setupCudaHook,
+    }:
+    prevAttrs: {
       # Patch the nvcc.profile.
       # Syntax:
       # - `=` for assignment,
@@ -131,38 +187,37 @@ attrsets.filterAttrs (attr: _: (builtins.hasAttr attr prev)) {
       # backend-stdenv.nix
 
       postPatch =
-        (oldAttrs.postPatch or "")
+        (prevAttrs.postPatch or "")
         + ''
           substituteInPlace bin/nvcc.profile \
-            --replace \
+            --replace-fail \
-              '$(TOP)/lib' \
-              "''${!outputLib}/lib" \
-            --replace \
               '$(TOP)/$(_NVVM_BRANCH_)' \
               "''${!outputBin}/nvvm" \
-            --replace \
+            --replace-fail \
               '$(TOP)/$(_TARGET_DIR_)/include' \
               "''${!outputDev}/include"
 
           cat << EOF >> bin/nvcc.profile
 
           # Fix a compatible backend compiler
-          PATH += ${lib.getBin cc}/bin:
+          PATH += "${backendStdenv.cc}/bin":
 
           # Expose the split-out nvvm
-          LIBRARIES =+ -L''${!outputBin}/nvvm/lib
+          LIBRARIES =+ "-L''${!outputBin}/nvvm/lib"
-          INCLUDES =+ -I''${!outputBin}/nvvm/include
+          INCLUDES =+ "-I''${!outputBin}/nvvm/include"
-
-          # Expose cudart and the libcuda stubs
-          LIBRARIES =+ -L$static/lib" "-L${final.cuda_cudart.lib}/lib -L${final.cuda_cudart.lib}/lib/stubs
-          INCLUDES =+ -I${final.cuda_cudart.dev}/include
           EOF
         '';
 
-      propagatedBuildInputs = [ final.setupCudaHook ];
+      # NOTE(@connorbaker):
+      # Though it might seem odd or counter-intuitive to add the setup hook to `propagatedBuildInputs` instead of
+      # `propagatedNativeBuildInputs`, it is necessary! If you move the setup hook from `propagatedBuildInputs` to
+      # `propagatedNativeBuildInputs`, it stops being propagated to downstream packages during their build because
+      # setup hooks in `propagatedNativeBuildInputs` are not designed to affect the runtime or build environment of
+      # dependencies; they are only meant to affect the build environment of the package that directly includes them.
+      propagatedBuildInputs = (prevAttrs.propagatedBuildInputs or [ ]) ++ [ setupCudaHook ];
 
       postInstall =
-        (oldAttrs.postInstall or "")
+        (prevAttrs.postInstall or "")
         + ''
           moveToOutput "nvvm" "''${!outputBin}"
         '';
@@ -170,48 +225,77 @@ attrsets.filterAttrs (attr: _: (builtins.hasAttr attr prev)) {
       # The nvcc and cicc binaries contain hard-coded references to /usr
       allowFHSReferences = true;
 
-      meta = (oldAttrs.meta or { }) // {
+      meta = (prevAttrs.meta or { }) // {
         mainProgram = "nvcc";
       };
-    }
+    };
-  );
 
-  cuda_nvprof = prev.cuda_nvprof.overrideAttrs (prevAttrs: {
+  cuda_nvprof =
-    buildInputs = prevAttrs.buildInputs ++ [ final.cuda_cupti.lib ];
+    { cuda_cupti }: prevAttrs: { buildInputs = prevAttrs.buildInputs ++ [ cuda_cupti.lib ]; };
-  });
 
-  cuda_demo_suite = addBuildInputs prev.cuda_demo_suite [
+  cuda_demo_suite =
-    final.pkgs.freeglut
+    {
-    final.pkgs.libGLU
+      freeglut,
-    final.pkgs.libglvnd
+      libcufft,
-    final.pkgs.mesa
+      libcurand,
-    final.libcufft.lib
+      libGLU,
-    final.libcurand.lib
+      libglvnd,
-  ];
+      mesa,
+    }:
+    prevAttrs: {
+      buildInputs = prevAttrs.buildInputs ++ [
+        freeglut
+        libcufft.lib
+        libcurand.lib
+        libGLU
+        libglvnd
+        mesa
+      ];
+    };
 
-  nsight_compute = prev.nsight_compute.overrideAttrs (prevAttrs: {
+  nsight_compute =
-    nativeBuildInputs =
+    {
-      prevAttrs.nativeBuildInputs
+      lib,
-      ++ (
+      qt5 ? null,
-        if (strings.versionOlder prev.nsight_compute.version "2022.2.0") then
+      qt6 ? null,
-          [ final.pkgs.qt5.wrapQtAppsHook ]
+    }:
-        else
-          [ final.pkgs.qt6.wrapQtAppsHook ]
-      );
-    buildInputs =
-      prevAttrs.buildInputs
-      ++ (
-        if (strings.versionOlder prev.nsight_compute.version "2022.2.0") then
-          [ final.pkgs.qt5.qtwebview ]
-        else
-          [ final.pkgs.qt6.qtwebview ]
-      );
-  });
-
-  nsight_systems = prev.nsight_systems.overrideAttrs (
     prevAttrs:
     let
-      qt = if lib.versionOlder prevAttrs.version "2022.4.2.1" then final.pkgs.qt5 else final.pkgs.qt6;
+      inherit (lib.strings) versionOlder versionAtLeast;
+      inherit (prevAttrs) version;
+      qt = if versionOlder version "2022.2.0" then qt5 else qt6;
+      inherit (qt) wrapQtAppsHook qtwebview;
+    in
+    {
+      nativeBuildInputs = prevAttrs.nativeBuildInputs ++ [ wrapQtAppsHook ];
+      buildInputs = prevAttrs.buildInputs ++ [ qtwebview ];
+      brokenConditions = prevAttrs.brokenConditions // {
+        "Qt 5 missing (<2022.2.0)" = !(versionOlder version "2022.2.0" -> qt5 != null);
+        "Qt 6 missing (>=2022.2.0)" = !(versionAtLeast version "2022.2.0" -> qt6 != null);
+      };
+    };
+
+  nsight_systems =
+    {
+      cuda_cudart,
+      cudaOlder,
+      gst_all_1,
+      lib,
+      nss,
+      numactl,
+      pulseaudio,
+      qt5 ? null,
+      qt6 ? null,
+      rdma-core,
+      ucx,
+      wayland,
+      xorg,
+    }:
+    prevAttrs:
+    let
+      inherit (lib.strings) versionOlder versionAtLeast;
+      inherit (prevAttrs) version;
+      qt = if lib.strings.versionOlder prevAttrs.version "2022.4.2.1" then qt5 else qt6;
       qtwayland =
         if lib.versions.major qt.qtbase.version == "5" then
           lib.getBin qt.qtwayland
@@ -223,55 +307,57 @@ attrsets.filterAttrs (attr: _: (builtins.hasAttr attr prev)) {
       # An ad hoc replacement for
       # https://github.com/ConnorBaker/cuda-redist-find-features/issues/11
       env.rmPatterns = toString [
+        "nsight-systems/*/*/lib{arrow,jpeg}*"
+        "nsight-systems/*/*/lib{ssl,ssh,crypto}*"
+        "nsight-systems/*/*/libboost*"
+        "nsight-systems/*/*/libexec"
         "nsight-systems/*/*/libQt*"
         "nsight-systems/*/*/libstdc*"
-        "nsight-systems/*/*/libboost*"
-        "nsight-systems/*/*/lib{ssl,ssh,crypto}*"
-        "nsight-systems/*/*/lib{arrow,jpeg}*"
         "nsight-systems/*/*/Mesa"
-        "nsight-systems/*/*/python/bin/python"
-        "nsight-systems/*/*/libexec"
         "nsight-systems/*/*/Plugins"
+        "nsight-systems/*/*/python/bin/python"
       ];
       postPatch =
         prevAttrs.postPatch or ""
         + ''
-          for path in $rmPatterns ; do
+          for path in $rmPatterns; do
             rm -r "$path"
           done
         '';
       nativeBuildInputs = prevAttrs.nativeBuildInputs ++ [ qt.wrapQtAppsHook ];
       buildInputs = prevAttrs.buildInputs ++ [
-        final.cuda_cudart.stubs
-        final.pkgs.alsa-lib
-        final.pkgs.boost178
-        final.pkgs.e2fsprogs
-        final.pkgs.gst_all_1.gst-plugins-base
-        final.pkgs.gst_all_1.gstreamer
-        final.pkgs.nss
-        final.pkgs.numactl
-        final.pkgs.pulseaudio
-        final.pkgs.rdma-core
-        final.pkgs.ucx
-        final.pkgs.wayland
-        final.pkgs.xorg.libXcursor
-        final.pkgs.xorg.libXdamage
-        final.pkgs.xorg.libXrandr
-        final.pkgs.xorg.libXtst
-        qt.qtbase
         (qt.qtdeclarative or qt.full)
         (qt.qtsvg or qt.full)
+        cuda_cudart.stubs
+        gst_all_1.gst-plugins-base
+        gst_all_1.gstreamer
+        nss
+        numactl
+        pulseaudio
+        qt.qtbase
         qtWaylandPlugins
+        rdma-core
+        ucx
+        wayland
+        xorg.libXcursor
+        xorg.libXdamage
+        xorg.libXrandr
+        xorg.libXtst
       ];
 
-      # Older releases require boost 1.70 deprecated in Nixpkgs
+      brokenConditions = prevAttrs.brokenConditions // {
-      meta.broken = prevAttrs.meta.broken or false || lib.versionOlder final.cudaVersion "11.8";
+        # Older releases require boost 1.70, which is deprecated in Nixpkgs
-    }
+        "CUDA too old (<11.8)" = cudaOlder "11.8";
-  );
+        "Qt 5 missing (<2022.4.2.1)" = !(versionOlder version "2022.4.2.1" -> qt5 != null);
+        "Qt 6 missing (>=2022.4.2.1)" = !(versionAtLeast version "2022.4.2.1" -> qt6 != null);
+      };
+    };
 
-  nvidia_driver = prev.nvidia_driver.overrideAttrs {
+  nvidia_driver =
-    # No need to support this package as we have drivers already
+    { }:
-    # in linuxPackages.
+    prevAttrs: {
-    meta.broken = true;
+      brokenConditions = prevAttrs.brokenConditions // {
-  };
+        "Package is not supported; use drivers from linuxPackages" = true;
+      };
+    };
 }

pkgs/development/cuda-modules/cutensor/extension.nix

@@ -15,9 +15,9 @@
 {
   cudaVersion,
   flags,
-  hostPlatform,
   lib,
   mkVersionedPackageName,
+  stdenv,
 }:
 let
   inherit (lib)
@@ -29,6 +29,8 @@ let
     trivial
     ;
 
+  inherit (stdenv) hostPlatform;
+
   redistName = "cutensor";
   pname = "libcutensor";
 

pkgs/development/cuda-modules/flags.nix

@@ -7,7 +7,7 @@
   cudaForwardCompat ? (config.cudaForwardCompat or true),
   lib,
   cudaVersion,
-  hostPlatform,
+  stdenv,
   # gpus :: List Gpu
   gpus,
 }:
@@ -20,6 +20,8 @@ let
     trivial
     ;
 
+  inherit (stdenv) hostPlatform;
+
   # Flags are determined based on your CUDA toolkit by default.  You may benefit
   # from improved performance, reduced file size, or greater hardware support by
   # passing a configuration based on your specific GPU environment.
@@ -207,6 +209,11 @@ let
       # E.g. "-gencode=arch=compute_75,code=sm_75 ... -gencode=arch=compute_86,code=compute_86"
       gencodeString = strings.concatStringsSep " " gencode;
 
+      # cmakeCudaArchitecturesString :: String
+      # A semicolon-separated string of CUDA capabilities without dots, suitable for passing to CMake.
+      # E.g. "75;86"
+      cmakeCudaArchitecturesString = strings.concatMapStringsSep ";" dropDot cudaCapabilities;
+
       # Jetson devices cannot be targeted by the same binaries which target non-Jetson devices. While
       # NVIDIA provides both `linux-aarch64` and `linux-sbsa` packages, which both target `aarch64`,
       # they are built with different settings and cannot be mixed.
@@ -270,6 +277,8 @@ assert
       ];
       gencodeString = "-gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_86,code=sm_86 -gencode=arch=compute_86,code=compute_86";
 
+      cmakeCudaArchitecturesString = "75;86";
+
       isJetsonBuild = false;
     };
     actual = formatCapabilities {
@@ -339,6 +348,8 @@ assert
       ];
       gencodeString = "-gencode=arch=compute_62,code=sm_62 -gencode=arch=compute_72,code=sm_72 -gencode=arch=compute_72,code=compute_72";
 
+      cmakeCudaArchitecturesString = "62;72";
+
       isJetsonBuild = true;
     };
     actual = formatCapabilities {

pkgs/development/cuda-modules/generic-builders/manifest.nix

@@ -10,7 +10,6 @@
   markForCudatoolkitRootHook,
   flags,
   stdenv,
-  hostPlatform,
   # Builder-specific arguments
   # Short package name (e.g., "cuda_cccl")
   # pname : String
@@ -40,6 +39,8 @@ let
     sourceTypes
     ;
 
+  inherit (stdenv) hostPlatform;
+
   # Get the redist architectures for which package provides distributables.
   # These are used by meta.platforms.
   supportedRedistArchs = builtins.attrNames featureRelease;
@@ -48,7 +49,7 @@ let
   # It is `"unsupported"` if the redistributable is not supported on the target platform.
   redistArch = flags.getRedistArch hostPlatform.system;
 
-  sourceMatchesHost = flags.getNixSystem redistArch == stdenv.hostPlatform.system;
+  sourceMatchesHost = flags.getNixSystem redistArch == hostPlatform.system;
 in
 backendStdenv.mkDerivation (finalAttrs: {
   # NOTE: Even though there's no actual buildPhase going on here, the derivations of the
@@ -127,7 +128,18 @@ backendStdenv.mkDerivation (finalAttrs: {
   # brokenConditions :: AttrSet Bool
   # Sets `meta.broken = true` if any of the conditions are true.
   # Example: Broken on a specific version of CUDA or when a dependency has a specific version.
-  brokenConditions = { };
+  brokenConditions = {
+    # Unclear how this is handled by Nix internals.
+    "Duplicate entries in outputs" = finalAttrs.outputs != lists.unique finalAttrs.outputs;
+    # Typically this results in the static output being empty, as all libraries are moved
+    # back to the lib output.
+    "lib output follows static output" =
+      let
+        libIndex = lists.findFirstIndex (x: x == "lib") null finalAttrs.outputs;
+        staticIndex = lists.findFirstIndex (x: x == "static") null finalAttrs.outputs;
+      in
+      libIndex != null && staticIndex != null && libIndex > staticIndex;
+  };
 
   # badPlatformsConditions :: AttrSet Bool
   # Sets `meta.badPlatforms = meta.platforms` if any of the conditions are true.
@@ -137,44 +149,43 @@ backendStdenv.mkDerivation (finalAttrs: {
   };
 
   # src :: Optional Derivation
-  src = trivial.pipe redistArch [
+  # If redistArch doesn't exist in redistribRelease, return null.
-    # If redistArch doesn't exist in redistribRelease, return null.
+  src = trivial.mapNullable (
-    (redistArch: redistribRelease.${redistArch} or null)
+    { relative_path, sha256, ... }:
-    # If the release is non-null, fetch the source; otherwise, return null.
+    fetchurl {
-    (trivial.mapNullable (
+      url = "https://developer.download.nvidia.com/compute/${redistName}/redist/${relative_path}";
-      { relative_path, sha256, ... }:
+      inherit sha256;
-      fetchurl {
+    }
-        url = "https://developer.download.nvidia.com/compute/${redistName}/redist/${relative_path}";
+  ) (redistribRelease.${redistArch} or null);
-        inherit sha256;
-      }
-    ))
-  ];
-
-  # Handle the pkg-config files:
-  # 1. No FHS
-  # 2. Location expected by the pkg-config wrapper
-  # 3. Generate unversioned names too
-  postPatch = ''
-    for path in pkg-config pkgconfig ; do
-      [[ -d "$path" ]] || continue
-      mkdir -p share/pkgconfig
-      mv "$path"/* share/pkgconfig/
-      rmdir "$path"
-    done
-
-    for pc in share/pkgconfig/*.pc ; do
-      sed -i \
-        -e "s|^cudaroot\s*=.*\$|cudaroot=''${!outputDev}|" \
-        -e "s|^libdir\s*=.*/lib\$|libdir=''${!outputLib}/lib|" \
-        -e "s|^includedir\s*=.*/include\$|includedir=''${!outputDev}/include|" \
-        "$pc"
-    done
 
+  postPatch =
+    # Pkg-config's setup hook expects configuration files in $out/share/pkgconfig
+    ''
+      for path in pkg-config pkgconfig; do
+        [[ -d "$path" ]] || continue
+        mkdir -p share/pkgconfig
+        mv "$path"/* share/pkgconfig/
+        rmdir "$path"
+      done
+    ''
+    # Rewrite FHS paths with store paths
+    # NOTE: output* fall back to out if the corresponding output isn't defined.
+    + ''
+      for pc in share/pkgconfig/*.pc; do
+        sed -i \
+          -e "s|^cudaroot\s*=.*\$|cudaroot=''${!outputDev}|" \
+          -e "s|^libdir\s*=.*/lib\$|libdir=''${!outputLib}/lib|" \
+          -e "s|^includedir\s*=.*/include\$|includedir=''${!outputDev}/include|" \
+          "$pc"
+      done
+    ''
+    # Generate unversioned names.
     # E.g. cuda-11.8.pc -> cuda.pc
-    for pc in share/pkgconfig/*-"$majorMinorVersion.pc" ; do
+    + ''
-      ln -s "$(basename "$pc")" "''${pc%-$majorMinorVersion.pc}".pc
+      for pc in share/pkgconfig/*-"$majorMinorVersion.pc"; do
-    done
+        ln -s "$(basename "$pc")" "''${pc%-$majorMinorVersion.pc}".pc
-  '';
+      done
+    '';
 
   env.majorMinorVersion = cudaMajorMinorVersion;
 
@@ -233,7 +244,7 @@ backendStdenv.mkDerivation (finalAttrs: {
     # Handle the existence of libPath, which requires us to re-arrange the lib directory
     + strings.optionalString (libPath != null) ''
       full_lib_path="lib/${libPath}"
-      if [[ ! -d "$full_lib_path" ]] ; then
+      if [[ ! -d "$full_lib_path" ]]; then
         echo "${finalAttrs.pname}: '$full_lib_path' does not exist, only found:" >&2
         find lib/ -mindepth 1 -maxdepth 1 >&2
         echo "This release might not support your CUDA version" >&2
@@ -264,9 +275,9 @@ backendStdenv.mkDerivation (finalAttrs: {
   postInstallCheck = ''
     echo "Executing postInstallCheck"
 
-    if [[ -z "''${allowFHSReferences-}" ]] ; then
+    if [[ -z "''${allowFHSReferences-}" ]]; then
       mapfile -t outputPaths < <(for o in $(getAllOutputNames); do echo "''${!o}"; done)
-      if grep --max-count=5 --recursive --exclude=LICENSE /usr/ "''${outputPaths[@]}" ; then
+      if grep --max-count=5 --recursive --exclude=LICENSE /usr/ "''${outputPaths[@]}"; then
         echo "Detected references to /usr" >&2
         exit 1
       fi

pkgs/development/cuda-modules/generic-builders/multiplex.nix

@@ -3,7 +3,7 @@
   lib,
   cudaVersion,
   flags,
-  hostPlatform,
+  stdenv,
   # Expected to be passed by the caller
   mkVersionedPackageName,
   # pname :: String
@@ -40,6 +40,8 @@ let
     strings
     ;
 
+  inherit (stdenv) hostPlatform;
+
   evaluatedModules = modules.evalModules {
     modules = [
       ../modules

pkgs/development/cuda-modules/nccl/default.nix

@@ -17,9 +17,10 @@ let
     cuda_cccl
     cuda_cudart
     cuda_nvcc
+    cudaAtLeast
     cudaFlags
+    cudaOlder
     cudatoolkit
-    cudaVersion
     ;
 in
 backendStdenv.mkDerivation (finalAttrs: {
@@ -33,6 +34,7 @@ backendStdenv.mkDerivation (finalAttrs: {
     hash = "sha256-IF2tILwW8XnzSmfn7N1CO7jXL95gUp02guIW5n1eaig=";
   };
 
+  __structuredAttrs = true;
   strictDeps = true;
 
   outputs = [
@@ -46,12 +48,12 @@ backendStdenv.mkDerivation (finalAttrs: {
       autoAddDriverRunpath
       python3
     ]
-    ++ lib.optionals (lib.versionOlder cudaVersion "11.4") [ cudatoolkit ]
+    ++ lib.optionals (cudaOlder "11.4") [ cudatoolkit ]
-    ++ lib.optionals (lib.versionAtLeast cudaVersion "11.4") [ cuda_nvcc ];
+    ++ lib.optionals (cudaAtLeast "11.4") [ cuda_nvcc ];
 
   buildInputs =
-    lib.optionals (lib.versionOlder cudaVersion "11.4") [ cudatoolkit ]
+    lib.optionals (cudaOlder "11.4") [ cudatoolkit ]
-    ++ lib.optionals (lib.versionAtLeast cudaVersion "11.4") [
+    ++ lib.optionals (cudaAtLeast "11.4") [
       cuda_nvcc.dev # crt/host_config.h
       cuda_cudart
     ]
@@ -59,25 +61,25 @@ backendStdenv.mkDerivation (finalAttrs: {
     # against other version, like below, it's important that we use the same format. Otherwise,
     # we'll get incorrect results.
     # For example, lib.versionAtLeast "12.0" "12.0.0" == false.
-    ++ lib.optionals (lib.versionAtLeast cudaVersion "12.0") [ cuda_cccl ];
+    ++ lib.optionals (cudaAtLeast "12.0") [ cuda_cccl ];
 
   env.NIX_CFLAGS_COMPILE = toString [ "-Wno-unused-function" ];
 
-  preConfigure = ''
+  postPatch = ''
     patchShebangs ./src/device/generate.py
-    makeFlagsArray+=(
-      "NVCC_GENCODE=${lib.concatStringsSep " " cudaFlags.gencode}"
-    )
   '';
 
-  makeFlags =
+  makeFlagsArray =
-    [ "PREFIX=$(out)" ]
+    [
-    ++ lib.optionals (lib.versionOlder cudaVersion "11.4") [
+      "PREFIX=$(out)"
+      "NVCC_GENCODE=${cudaFlags.gencodeString}"
+    ]
+    ++ lib.optionals (cudaOlder "11.4") [
       "CUDA_HOME=${cudatoolkit}"
       "CUDA_LIB=${lib.getLib cudatoolkit}/lib"
       "CUDA_INC=${lib.getDev cudatoolkit}/include"
     ]
-    ++ lib.optionals (lib.versionAtLeast cudaVersion "11.4") [
+    ++ lib.optionals (cudaAtLeast "11.4") [
       "CUDA_HOME=${cuda_nvcc}"
       "CUDA_LIB=${lib.getLib cuda_cudart}/lib"
       "CUDA_INC=${lib.getDev cuda_cudart}/include"

pkgs/development/cuda-modules/saxpy/default.nix

@@ -10,8 +10,9 @@ let
     cuda_cccl
     cuda_cudart
     cuda_nvcc
+    cudaAtLeast
+    cudaOlder
     cudatoolkit
-    cudaVersion
     flags
     libcublas
     setupCudaHook
@@ -24,6 +25,7 @@ backendStdenv.mkDerivation {
 
   src = ./.;
 
+  __structuredAttrs = true;
   strictDeps = true;
 
   nativeBuildInputs =
@@ -31,24 +33,22 @@ backendStdenv.mkDerivation {
       cmake
       autoAddDriverRunpath
     ]
-    ++ lib.optionals (lib.versionOlder cudaVersion "11.4") [ cudatoolkit ]
+    ++ lib.optionals (cudaOlder "11.4") [ cudatoolkit ]
-    ++ lib.optionals (lib.versionAtLeast cudaVersion "11.4") [ cuda_nvcc ];
+    ++ lib.optionals (cudaAtLeast "11.4") [ cuda_nvcc ];
 
   buildInputs =
-    lib.optionals (lib.versionOlder cudaVersion "11.4") [ cudatoolkit ]
+    lib.optionals (cudaOlder "11.4") [ cudatoolkit ]
-    ++ lib.optionals (lib.versionAtLeast cudaVersion "11.4") [
+    ++ lib.optionals (cudaAtLeast "11.4") [
       (getDev libcublas)
       (getLib libcublas)
       (getOutput "static" libcublas)
       cuda_cudart
     ]
-    ++ lib.optionals (lib.versionAtLeast cudaVersion "12.0") [ cuda_cccl ];
+    ++ lib.optionals (cudaAtLeast "12.0") [ cuda_cccl ];
 
-  cmakeFlags = [
+  cmakeFlagsArray = [
     (lib.cmakeBool "CMAKE_VERBOSE_MAKEFILE" true)
-    (lib.cmakeFeature "CMAKE_CUDA_ARCHITECTURES" (
+    (lib.cmakeFeature "CMAKE_CUDA_ARCHITECTURES" flags.cmakeCudaArchitecturesString)
-      with flags; lib.concatStringsSep ";" (lib.lists.map dropDot cudaCapabilities)
-    ))
   ];
 
   meta = rec {
@@ -56,6 +56,6 @@ backendStdenv.mkDerivation {
     license = lib.licenses.mit;
     maintainers = lib.teams.cuda.members;
     platforms = lib.platforms.unix;
-    badPlatforms = lib.optionals flags.isJetsonBuild platforms;
+    badPlatforms = lib.optionals (flags.isJetsonBuild && cudaOlder "11.4") platforms;
   };
 }

pkgs/development/cuda-modules/setup-hooks/mark-for-cudatoolkit-root-hook.sh

@@ -1,14 +1,25 @@
 # shellcheck shell=bash
 
-# Should we mimick cc-wrapper's "hygiene"?
+(( ${hostOffset:?} == -1 && ${targetOffset:?} == 0)) || return 0
-[[ -z ${strictDeps-} ]] || (( "$hostOffset" < 0 )) || return 0
 
 echo "Sourcing mark-for-cudatoolkit-root-hook" >&2
 
 markForCUDAToolkit_ROOT() {
-    mkdir -p "${prefix}/nix-support"
+    mkdir -p "${prefix:?}/nix-support"
-    [[ -f "${prefix}/nix-support/include-in-cudatoolkit-root" ]] && return
+    local markerPath="$prefix/nix-support/include-in-cudatoolkit-root"
-    echo "$pname-$output" > "${prefix}/nix-support/include-in-cudatoolkit-root"
+
+    # Return early if the file already exists.
+    [[ -f "$markerPath" ]] && return 0
+
+    # Always create the file, even if it's empty, since setup-cuda-hook relies on its existence.
+    # However, only populate it if strictDeps is not set.
+    touch "$markerPath"
+
+    # Return early if strictDeps is set.
+    [[ -n "${strictDeps-}" ]] && return 0
+
+    # Populate the file with the package name and output.
+    echo "${pname:?}-${output:?}" > "$markerPath"
 }
 
 fixupOutputHooks+=(markForCUDAToolkit_ROOT)

pkgs/development/cuda-modules/setup-hooks/setup-cuda-hook.sh

@@ -9,7 +9,7 @@ reason=
 [[ -n ${cudaSetupHookOnce-} ]] && guard=Skipping && reason=" because the hook has been propagated more than once"
 
 if (( "${NIX_DEBUG:-0}" >= 1 )) ; then
-    echo "$guard hostOffset=$hostOffset targetOffset=$targetOffset setupCudaHook$reason" >&2
+    echo "$guard hostOffset=$hostOffset targetOffset=$targetOffset setup-cuda-hook$reason" >&2
 else
     echo "$guard setup-cuda-hook$reason" >&2
 fi
@@ -24,16 +24,19 @@ extendcudaHostPathsSeen() {
     (( "${NIX_DEBUG:-0}" >= 1 )) && echo "extendcudaHostPathsSeen $1" >&2
 
     local markerPath="$1/nix-support/include-in-cudatoolkit-root"
-    [[ ! -f "${markerPath}" ]] && return
+    [[ ! -f "${markerPath}" ]] && return 0
-    [[ -v cudaHostPathsSeen[$1] ]] && return
+    [[ -v cudaHostPathsSeen[$1] ]] && return 0
 
     cudaHostPathsSeen["$1"]=1
 
     # E.g. cuda_cudart-lib
     local cudaOutputName
-    read -r cudaOutputName < "$markerPath"
+    # Fail gracefully if the file is empty.
+    # One reason the file may be empty: the package was built with strictDeps set, but the current build does not have
+    # strictDeps set.
+    read -r cudaOutputName < "$markerPath" || return 0
 
-    [[ -z "$cudaOutputName" ]] && return
+    [[ -z "$cudaOutputName" ]] && return 0
 
     local oldPath="${cudaOutputToPath[$cudaOutputName]-}"
     [[ -n "$oldPath" ]] && echo "extendcudaHostPathsSeen: warning: overwriting $cudaOutputName from $oldPath to $1" >&2
@@ -59,7 +62,7 @@ setupCUDAToolkitCompilers() {
     echo Executing setupCUDAToolkitCompilers >&2
 
     if [[ -n "${dontSetupCUDAToolkitCompilers-}" ]] ; then
-        return
+        return 0
     fi
 
     # Point NVCC at a compatible compiler
@@ -99,7 +102,7 @@ preConfigureHooks+=(setupCUDAToolkitCompilers)
 propagateCudaLibraries() {
     (( "${NIX_DEBUG:-0}" >= 1 )) && echo "propagateCudaLibraries: cudaPropagateToOutput=$cudaPropagateToOutput cudaHostPathsSeen=${!cudaHostPathsSeen[*]}" >&2
 
-    [[ -z "${cudaPropagateToOutput-}" ]] && return
+    [[ -z "${cudaPropagateToOutput-}" ]] && return 0
 
     mkdir -p "${!cudaPropagateToOutput}/nix-support"
     # One'd expect this should be propagated-bulid-build-deps, but that doesn't seem to work

pkgs/development/cuda-modules/tensorrt/fixup.nix

@@ -1,12 +1,12 @@
 {
   cudaVersion,
   final,
-  hostPlatform,
   lib,
   mkVersionedPackageName,
   package,
   patchelf,
   requireFile,
+  stdenv,
   ...
 }:
 let
@@ -17,6 +17,7 @@ let
     strings
     versions
     ;
+  inherit (stdenv) hostPlatform;
   # targetArch :: String
   targetArch = attrsets.attrByPath [ hostPlatform.system ] "unsupported" {
     x86_64-linux = "x86_64-linux-gnu";

pkgs/top-level/cuda-packages.nix

@@ -90,7 +90,7 @@ let
     [
       (import ../development/cuda-modules/setup-hooks/extension.nix)
       (callPackage ../development/cuda-modules/cuda/extension.nix { inherit cudaVersion; })
-      (callPackage ../development/cuda-modules/cuda/overrides.nix { inherit cudaVersion; })
+      (import ../development/cuda-modules/cuda/overrides.nix)
       (callPackage ../development/cuda-modules/generic-builders/multiplex.nix {
         inherit cudaVersion flags mkVersionedPackageName;
         pname = "cudnn";