Merge pull request #302300 from Ma27/kernel-zstd

linux kernel: prefer zstd where possible

nixos/doc/manual/release-notes/rl-2405.section.md

@@ -53,6 +53,10 @@ Use `services.pipewire.extraConfig` or `services.pipewire.configPackages` for Pi
 - `system.etc.overlay.enable` option was added. If enabled, `/etc` is
   mounted via an overlayfs instead of being created by a custom perl script.
 
+- For each supporting version of the Linux kernel firmware blobs and kernel modules
+  are compressed with zstd. For firmware blobs this means an increase of 4.4% in size, however
+  a significantly higher decompression speed.
+
 - NixOS AMIs are now uploaded regularly to a new AWS Account.
   Instructions on how to use them can be found on <https://nixos.github.io/amis>.
   We are working on integration the data into the NixOS homepage.

nixos/modules/services/hardware/udev.nix

@@ -167,10 +167,16 @@ let
       mv etc/udev/hwdb.bin $out
     '';
 
-  compressFirmware = firmware: if (config.boot.kernelPackages.kernelAtLeast "5.3" && (firmware.compressFirmware or true)) then
+  compressFirmware = firmware:
-    pkgs.compressFirmwareXz firmware
+    let
-  else
+      inherit (config.boot.kernelPackages) kernelAtLeast;
-    id firmware;
+    in
+      if ! (firmware.compressFirmware or true) then
+        firmware
+      else
+        if kernelAtLeast "5.19" then pkgs.compressFirmwareZstd firmware
+        else if kernelAtLeast "5.3" then pkgs.compressFirmwareXz firmware
+        else firmware;
 
   # Udev has a 512-character limit for ENV{PATH}, so create a symlink
   # tree to work around this.

pkgs/build-support/kernel/compress-firmware-xz.nix

@@ -1,29 +0,0 @@
-{ runCommand, lib }:
-
-firmware:
-
-let
-  args = {
-    allowedRequisites = [];
-  } // lib.optionalAttrs (firmware ? meta) { inherit (firmware) meta; };
-in
-
-runCommand "${firmware.name}-xz" args ''
-  mkdir -p $out/lib
-  (cd ${firmware} && find lib/firmware -type d -print0) |
-      (cd $out && xargs -0 mkdir -v --)
-  (cd ${firmware} && find lib/firmware -type f -print0) |
-      (cd $out && xargs -0rtP "$NIX_BUILD_CORES" -n1 \
-          sh -c 'xz -9c -T1 -C crc32 --lzma2=dict=2MiB "${firmware}/$1" > "$1.xz"' --)
-  (cd ${firmware} && find lib/firmware -type l) | while read link; do
-      target="$(readlink "${firmware}/$link")"
-      if [ -f "${firmware}/$link" ]; then
-        ln -vs -- "''${target/^${firmware}/$out}.xz" "$out/$link.xz"
-      else
-        ln -vs -- "''${target/^${firmware}/$out}" "$out/$link"
-      fi
-  done
-
-  echo "Checking for broken symlinks:"
-  find -L $out -type l -print -execdir false -- '{}' '+'
-''

pkgs/build-support/kernel/compress-firmware.nix

@@ -0,0 +1,43 @@
+{ runCommand, lib, type ? "zstd", zstd }:
+
+firmware:
+
+let
+  compressor = {
+    xz = {
+      ext = "xz";
+      nativeBuildInputs = [ ];
+      cmd = file: target: ''xz -9c -T1 -C crc32 --lzma2=dict=2MiB "${file}" > "${target}"'';
+    };
+    zstd = {
+      ext = "zst";
+      nativeBuildInputs = [ zstd ];
+      cmd = file: target: ''zstd -T1 -19 --long --check -f "${file}" -o "${target}"'';
+    };
+  }.${type} or (throw "Unsupported compressor type for firmware.");
+
+  args = {
+    allowedRequisites = [];
+    inherit (compressor) nativeBuildInputs;
+  } // lib.optionalAttrs (firmware ? meta) { inherit (firmware) meta; };
+in
+
+runCommand "${firmware.name}-${type}" args ''
+  mkdir -p $out/lib
+  (cd ${firmware} && find lib/firmware -type d -print0) |
+      (cd $out && xargs -0 mkdir -v --)
+  (cd ${firmware} && find lib/firmware -type f -print0) |
+      (cd $out && xargs -0rtP "$NIX_BUILD_CORES" -n1 \
+          sh -c '${compressor.cmd "${firmware}/$1" "$1.${compressor.ext}"}' --)
+  (cd ${firmware} && find lib/firmware -type l) | while read link; do
+      target="$(readlink "${firmware}/$link")"
+      if [ -f "${firmware}/$link" ]; then
+        ln -vs -- "''${target/^${firmware}/$out}.${compressor.ext}" "$out/$link.${compressor.ext}"
+      else
+        ln -vs -- "''${target/^${firmware}/$out}" "$out/$link"
+      fi
+  done
+
+  echo "Checking for broken symlinks:"
+  find -L $out -type l -print -execdir false -- '{}' '+'
+''

pkgs/build-support/kernel/modules-closure.sh

@@ -80,7 +80,7 @@ for module in $(< ~-/closure); do
     # of its output.
     modinfo -b $kernel --set-version "$version" -F firmware $module | grep -v '^name:' | while read -r i; do
         echo "firmware for $module: $i"
-        for name in "$i" "$i.xz" ""; do
+        for name in "$i" "$i.xz" "$i.zst" ""; do
             [ -z "$name" ] && echo "WARNING: missing firmware $i for module $module"
             if cp -v --parents --no-preserve=mode lib/firmware/$name "$out" 2>/dev/null; then
                 break

pkgs/build-support/vm/default.nix

@@ -40,10 +40,14 @@ rec {
         ${pkgs.stdenv.cc.libc}/lib/libc.so.* \
         ${pkgs.stdenv.cc.libc}/lib/libm.so.* \
         ${pkgs.stdenv.cc.libc}/lib/libresolv.so.* \
+        ${pkgs.stdenv.cc.libc}/lib/libpthread.so.* \
+        ${pkgs.zstd.out}/lib/libzstd.so.* \
+        ${pkgs.xz.out}/lib/liblzma.so.* \
         $out/lib
 
       # Copy BusyBox.
       cp -pd ${pkgs.busybox}/bin/* $out/bin
+      cp -pd ${pkgs.kmod}/bin/* $out/bin
 
       # Run patchelf to make the programs refer to the copied libraries.
       for i in $out/bin/* $out/lib/*; do if ! test -L $i; then nuke-refs $i; fi; done
@@ -54,6 +58,11 @@ rec {
               patchelf --set-interpreter $out/lib/ld-*.so.? --set-rpath $out/lib $i || true
           fi
       done
+
+      find $out/lib -type f \! -name 'ld*.so.?' | while read i; do
+        echo "patching $i..."
+        patchelf --set-rpath $out/lib $i
+      done
     ''; # */
 
 

pkgs/os-specific/linux/kernel/common-config.nix

@@ -871,12 +871,14 @@ let
     };
 
     zram = {
-      ZRAM            = module;
+      ZRAM                          = module;
-      ZRAM_WRITEBACK  = option yes;
+      ZRAM_WRITEBACK                = option yes;
-      ZRAM_MULTI_COMP = whenAtLeast "6.2" yes;
+      ZRAM_MULTI_COMP               = whenAtLeast "6.2" yes;
-      ZSWAP           = option yes;
+      ZRAM_DEF_COMP_ZSTD            = whenAtLeast "5.11" yes;
-      ZPOOL           = yes;
+      ZSWAP                         = option yes;
-      ZBUD            = option yes;
+      ZSWAP_COMPRESSOR_DEFAULT_ZSTD = whenAtLeast "5.7" yes;
+      ZPOOL                         = yes;
+      ZSMALLOC                      = option yes;
     };
 
     brcmfmac = {
@@ -948,8 +950,9 @@ let
       THRUSTMASTER_FF    = yes;
       ZEROPLUS_FF        = yes;
 
-      MODULE_COMPRESS    = whenOlder "5.13" yes;
+      MODULE_COMPRESS      = whenOlder "5.13" yes;
-      MODULE_COMPRESS_XZ = yes;
+      MODULE_COMPRESS_XZ   = whenOlder "5.13" yes;
+      MODULE_COMPRESS_ZSTD = whenAtLeast "5.13" yes;
 
       SYSVIPC            = yes;  # System-V IPC
 
@@ -1124,6 +1127,7 @@ let
       FW_LOADER_USER_HELPER_FALLBACK = option no;
 
       FW_LOADER_COMPRESS = whenAtLeast "5.3" yes;
+      FW_LOADER_COMPRESS_ZSTD = whenAtLeast "5.19" yes;
 
       HOTPLUG_PCI_ACPI = yes; # PCI hotplug using ACPI
       HOTPLUG_PCI_PCIE = yes; # PCI-Expresscard hotplug support

pkgs/top-level/all-packages.nix

@@ -1302,7 +1302,9 @@ with pkgs;
 
   makeBinaryWrapper = callPackage ../build-support/setup-hooks/make-binary-wrapper { };
 
-  compressFirmwareXz = callPackage ../build-support/kernel/compress-firmware-xz.nix { };
+  compressFirmwareXz = callPackage ../build-support/kernel/compress-firmware.nix { type = "xz"; };
+
+  compressFirmwareZstd = callPackage ../build-support/kernel/compress-firmware.nix { type = "zstd"; };
 
   makeModulesClosure = { kernel, firmware, rootModules, allowMissing ? false }:
     callPackage ../build-support/kernel/modules-closure.nix {