Merge pull request #269452 from RaitoBezarius/zfs-superstable

.github/CODEOWNERS

@@ -322,9 +322,8 @@ pkgs/applications/version-management/forgejo @bendlas @emilylange
 /pkgs/development/ocaml-modules     @ulrikstrid
 
 # ZFS
-pkgs/os-specific/linux/zfs                @raitobezarius
-nixos/lib/make-single-disk-zfs-image.nix  @raitobezarius
-nixos/lib/make-multi-disk-zfs-image.nix   @raitobezarius
+pkgs/os-specific/linux/zfs/2_1.nix        @raitobezarius
+pkgs/os-specific/linux/zfs/generic.nix    @raitobezarius
 nixos/modules/tasks/filesystems/zfs.nix   @raitobezarius
 nixos/tests/zfs.nix                       @raitobezarius
 

nixos/modules/tasks/filesystems/zfs.nix

@@ -16,6 +16,7 @@ let
   cfgTrim = config.services.zfs.trim;
   cfgZED = config.services.zfs.zed;
 
+  selectModulePackage = package: config.boot.kernelPackages.${package.kernelModuleAttribute};
   inInitrd = any (fs: fs == "zfs") config.boot.initrd.supportedFilesystems;
   inSystem = any (fs: fs == "zfs") config.boot.supportedFilesystems;
 
@@ -210,11 +211,17 @@ in
   options = {
     boot.zfs = {
       package = mkOption {
-        readOnly = true;
         type = types.package;
-        default = if config.boot.zfs.enableUnstable then pkgs.zfsUnstable else pkgs.zfs;
-        defaultText = literalExpression "if config.boot.zfs.enableUnstable then pkgs.zfsUnstable else pkgs.zfs";
-        description = lib.mdDoc "Configured ZFS userland tools package.";
+        default = if cfgZfs.enableUnstable then pkgs.zfsUnstable else pkgs.zfs;
+        defaultText = literalExpression "if zfsUnstable is enabled then pkgs.zfsUnstable else pkgs.zfs";
+        description = lib.mdDoc "Configured ZFS userland tools package, use `pkgs.zfsUnstable` if you want to track the latest staging ZFS branch.";
+      };
+
+      modulePackage = mkOption {
+        internal = true; # It is supposed to be selected automatically, but can be overridden by expert users.
+        default = selectModulePackage cfgZfs.package;
+        type = types.package;
+        description = lib.mdDoc "Configured ZFS kernel module package.";
       };
 
       enabled = mkOption {
@@ -533,6 +540,10 @@ in
   config = mkMerge [
     (mkIf cfgZfs.enabled {
       assertions = [
+        {
+          assertion = cfgZfs.modulePackage.version == cfgZfs.package.version;
+          message = "The kernel module and the userspace tooling versions are not matching, this is an unsupported usecase.";
+        }
         {
           assertion = cfgZED.enableMail -> cfgZfs.package.enableMail;
           message = ''
@@ -571,18 +582,14 @@ in
         # https://github.com/NixOS/nixpkgs/issues/106093
         kernelParams = lib.optionals (!config.boot.zfs.allowHibernation) [ "nohibernate" ];
 
-        extraModulePackages = let
-          kernelPkg = if config.boot.zfs.enableUnstable then
-            config.boot.kernelPackages.zfsUnstable
-           else
-            config.boot.kernelPackages.zfs;
-        in [
-          (kernelPkg.override { inherit (cfgZfs) removeLinuxDRM; })
+        extraModulePackages = [
+          (cfgZfs.modulePackage.override { inherit (cfgZfs) removeLinuxDRM; })
         ];
       };
 
       boot.initrd = mkIf inInitrd {
-        kernelModules = [ "zfs" ] ++ optional (!cfgZfs.enableUnstable) "spl";
+        # spl has been removed in ≥ 2.2.0.
+        kernelModules = [ "zfs" ] ++ lib.optional (lib.versionOlder "2.2.0" version) "spl";
         extraUtilsCommands =
           mkIf (!config.boot.initrd.systemd.enable) ''
             copy_bin_and_libs ${cfgZfs.package}/sbin/zfs

nixos/tests/zfs.nix

@@ -13,6 +13,7 @@ let
                       else pkgs.linuxPackages
     , enableUnstable ? false
     , enableSystemdStage1 ? false
+    , zfsPackage ? if enableUnstable then pkgs.zfs else pkgs.zfsUnstable
     , extraTest ? ""
     }:
     makeTest {
@@ -21,7 +22,7 @@ let
         maintainers = [ adisbladis elvishjerricco ];
       };
 
-      nodes.machine = { pkgs, lib, ... }:
+      nodes.machine = { config, pkgs, lib, ... }:
         let
           usersharePath = "/var/lib/samba/usershares";
         in {
@@ -35,8 +36,8 @@ let
         boot.loader.efi.canTouchEfiVariables = true;
         networking.hostId = "deadbeef";
         boot.kernelPackages = kernelPackage;
+        boot.zfs.package = zfsPackage;
         boot.supportedFilesystems = [ "zfs" ];
-        boot.zfs.enableUnstable = enableUnstable;
         boot.initrd.systemd.enable = enableSystemdStage1;
 
         environment.systemPackages = [ pkgs.parted ];
@@ -193,6 +194,11 @@ let
 
 in {
 
+  # maintainer: @raitobezarius
+  series_2_1 = makeZfsTest "2.1-series" {
+    zfsPackage = pkgs.zfs_2_1;
+  };
+
   stable = makeZfsTest "stable" { };
 
   unstable = makeZfsTest "unstable" {

pkgs/os-specific/linux/zfs/2_1.nix

@@ -0,0 +1,49 @@
+{ callPackage
+, kernel ? null
+, stdenv
+, linuxKernel
+, removeLinuxDRM ? false
+, lib
+, nixosTests
+, fetchpatch
+, ...
+} @ args:
+
+let
+  stdenv' = if kernel == null then stdenv else kernel.stdenv;
+in
+callPackage ./generic.nix args {
+  # You have to ensure that in `pkgs/top-level/linux-kernels.nix`
+  # this attribute is the correct one for this package.
+  kernelModuleAttribute = "zfs_2_1";
+  # check the release notes for compatible kernels
+  kernelCompatible =
+    if stdenv'.isx86_64 || removeLinuxDRM
+    then kernel.kernelOlder "6.6"
+    else kernel.kernelOlder "6.2";
+
+  latestCompatibleLinuxPackages = if stdenv'.isx86_64 || removeLinuxDRM
+    then linuxKernel.packages.linux_6_5
+    else linuxKernel.packages.linux_6_1;
+
+  # This is a fixed version to the 2.1.x series, move only
+  # if the 2.1.x series moves.
+  version = "2.1.13";
+
+  extraPatches = [
+    (fetchpatch {
+      # https://github.com/openzfs/zfs/pull/15571
+      # Remove when it's backported to 2.1.x.
+      url = "https://github.com/robn/zfs/commit/617c990a4cf1157b0f8332f35672846ad16ca70a.patch";
+      hash = "sha256-j5YSrud7BaWk2npBl31qwFFLYltbut3CUjI1cjZOpag=";
+    })
+  ];
+
+  hash = "sha256-tqUCn/Hf/eEmyWRQthWQdmTJK2sDspnHiiEfn9rz2Kc=";
+
+  tests = [
+    nixosTests.zfs.series_2_1
+  ];
+
+  maintainers = [ lib.maintainers.raitobezarius ];
+}

pkgs/os-specific/linux/zfs/generic.nix

@@ -1,40 +1,47 @@
-{ pkgs, lib, stdenv, fetchFromGitHub, fetchpatch
-, autoreconfHook269, util-linux, nukeReferences, coreutils
-, perl, nixosTests
-, configFile ? "all"
-
-# Userspace dependencies
-, zlib, libuuid, python3, attr, openssl
-, libtirpc
-, nfs-utils, samba
-, gawk, gnugrep, gnused, systemd
-, smartmontools, enableMail ? false
-, sysstat, pkg-config
-, curl
-, pam
-
-# Kernel dependencies
-, kernel ? null
-, enablePython ? true
-, ...
-}:
-
-{ version
-, sha256
-, extraPatches ? []
-, rev ? "zfs-${version}"
-, isUnstable ? false
-, latestCompatibleLinuxPackages
-, kernelCompatible ? null
-}:
-
 let
-  inherit (lib) any optionalString optionals optional makeBinPath;
+  genericBuild =
+  { pkgs, lib, stdenv, fetchFromGitHub, fetchpatch
+  , autoreconfHook269, util-linux, nukeReferences, coreutils
+  , perl
+  , configFile ? "all"
+
+  # Userspace dependencies
+  , zlib, libuuid, python3, attr, openssl
+  , libtirpc
+  , nfs-utils, samba
+  , gawk, gnugrep, gnused, systemd
+  , smartmontools, enableMail ? false
+  , sysstat, pkg-config
+  , curl
+  , pam
+
+  # Kernel dependencies
+  , kernel ? null
+  , enablePython ? true
+  , ...
+  }@outerArgs:
+
+  assert (configFile == "kernel") -> (kernel != null);
+  { version
+  , hash
+  , kernelModuleAttribute
+  , extraPatches ? []
+  , rev ? "zfs-${version}"
+  , isUnstable ? false
+  , latestCompatibleLinuxPackages
+  , kernelCompatible ? null
+  , maintainers ? (with lib.maintainers; [ amarshall adamcstephens ])
+  , tests
+  }@innerArgs:
+
+  let
+    inherit (lib) any optionalString optionals optional makeBinPath versionAtLeast;
 
     smartmon = smartmontools.override { inherit enableMail; };
 
     buildKernel = any (n: n == configFile) [ "kernel" "all" ];
     buildUser = any (n: n == configFile) [ "user" "all" ];
+    isAtLeast22Series = versionAtLeast version "2.2.0";
 
     # XXX: You always want to build kernel modules with the same stdenv as the
     # kernel was built with. However, since zfs can also be built for userspace we
@@ -43,15 +50,17 @@ let
     # If you don't do this your ZFS builds will fail on any non-standard (e.g.
     # clang-built) kernels.
     stdenv' = if kernel == null then stdenv else kernel.stdenv;
-in
+  in
 
-stdenv'.mkDerivation {
+  stdenv'.mkDerivation {
     name = "zfs-${configFile}-${version}${optionalString buildKernel "-${kernel.version}"}";
+    pname = "zfs";
+    inherit version;
 
     src = fetchFromGitHub {
       owner = "openzfs";
       repo = "zfs";
-    inherit rev sha256;
+      inherit rev hash;
     };
 
     patches = extraPatches;
@@ -78,9 +87,19 @@ stdenv'.mkDerivation {
                                                     --replace "/etc/default"            "$out/etc/default"
       substituteInPlace ./contrib/initramfs/Makefile.am \
         --replace "/usr/share/initramfs-tools" "$out/usr/share/initramfs-tools"
+    '' + optionalString isAtLeast22Series ''
       substituteInPlace ./udev/vdev_id \
         --replace "PATH=/bin:/sbin:/usr/bin:/usr/sbin" \
          "PATH=${makeBinPath [ coreutils gawk gnused gnugrep systemd ]}"
+    '' + optionalString (!isAtLeast22Series) ''
+      substituteInPlace ./etc/zfs/Makefile.am --replace "\$(sysconfdir)/zfs" "$out/etc/zfs"
+
+      find ./contrib/initramfs -name Makefile.am -exec sed -i -e 's|/usr/share/initramfs-tools|'$out'/share/initramfs-tools|g' {} \;
+
+      substituteInPlace ./cmd/vdev_id/vdev_id \
+        --replace "PATH=/bin:/sbin:/usr/bin:/usr/sbin" \
+        "PATH=${makeBinPath [ coreutils gawk gnused gnugrep systemd ]}"
+    '' + ''
       substituteInPlace ./config/zfs-build.m4 \
         --replace "bashcompletiondir=/etc/bash_completion.d" \
           "bashcompletiondir=$out/share/bash-completion/completions"
@@ -180,23 +199,24 @@ stdenv'.mkDerivation {
     outputs = [ "out" ] ++ optionals buildUser [ "dev" ];
 
     passthru = {
-    inherit enableMail latestCompatibleLinuxPackages;
+      inherit enableMail latestCompatibleLinuxPackages kernelModuleAttribute;
+      # The corresponding userspace tools to this instantiation
+      # of the ZFS package set.
+      userspaceTools = genericBuild (outerArgs // {
+        configFile = "user";
+      }) innerArgs;
 
-    tests =
-      if isUnstable then [
-        nixosTests.zfs.unstable
-      ] else [
-        nixosTests.zfs.installer
-        nixosTests.zfs.stable
-      ];
+      inherit tests;
     };
 
     meta = {
-    description = "ZFS Filesystem Linux Kernel module";
+      description = "ZFS Filesystem Linux" + (if buildUser then " Userspace Tools" else " Kernel Module");
       longDescription = ''
         ZFS is a filesystem that combines a logical volume manager with a
         Copy-On-Write filesystem with data integrity detection and repair,
         snapshotting, cloning, block devices, deduplication, and more.
+
+        ${if buildUser then "This is the userspace tools package." else "This is the kernel module package."}
       '';
       homepage = "https://github.com/openzfs/zfs";
       changelog = "https://github.com/openzfs/zfs/releases/tag/zfs-${version}";
@@ -212,11 +232,12 @@ stdenv'.mkDerivation {
         with lib.systems.inspect.patterns;
         map (p: p // isLinux) ([ isx86_32 isx86_64 isPower isAarch64 isSparc ] ++ isArmv7);
 
-    maintainers = with lib.maintainers; [ jcumming jonringer globin raitobezarius ];
+      inherit maintainers;
       mainProgram = "zfs";
       # If your Linux kernel version is not yet supported by zfs, try zfsUnstable.
       # On NixOS set the option boot.zfs.enableUnstable.
       broken = buildKernel && (kernelCompatible != null) && !kernelCompatible;
     };
-}
-
+  };
+in
+  genericBuild

pkgs/os-specific/linux/zfs/stable.nix

@@ -4,6 +4,7 @@
 , linuxKernel
 , removeLinuxDRM ? false
 , fetchpatch
+, nixosTests
 , ...
 } @ args:
 
@@ -11,6 +12,9 @@ let
   stdenv' = if kernel == null then stdenv else kernel.stdenv;
 in
 callPackage ./generic.nix args {
+  # You have to ensure that in `pkgs/top-level/linux-kernels.nix`
+  # this attribute is the correct one for this package.
+  kernelModuleAttribute = "zfs";
   # check the release notes for compatible kernels
   kernelCompatible =
     if stdenv'.isx86_64 || removeLinuxDRM
@@ -24,5 +28,10 @@ callPackage ./generic.nix args {
   # this package should point to the latest release.
   version = "2.2.0";
 
-  sha256 = "sha256-s1sdXSrLu6uSOmjprbUa4cFsE2Vj7JX5i75e4vRnlvg=";
+  tests = [
+    nixosTests.zfs.installer
+    nixosTests.zfs.stable
+  ];
+
+  hash = "sha256-s1sdXSrLu6uSOmjprbUa4cFsE2Vj7JX5i75e4vRnlvg=";
 }

pkgs/os-specific/linux/zfs/unstable.nix

@@ -3,6 +3,7 @@
 , stdenv
 , linuxKernel
 , removeLinuxDRM ? false
+, nixosTests
 , ...
 } @ args:
 
@@ -10,6 +11,9 @@ let
   stdenv' = if kernel == null then stdenv else kernel.stdenv;
 in
 callPackage ./generic.nix args {
+  # You have to ensure that in `pkgs/top-level/linux-kernels.nix`
+  # this attribute is the correct one for this package.
+  kernelModuleAttribute = "zfsUnstable";
   # check the release notes for compatible kernels
   kernelCompatible = if stdenv'.isx86_64 || removeLinuxDRM
     then kernel.kernelOlder "6.6"
@@ -26,7 +30,10 @@ callPackage ./generic.nix args {
   version = "2.2.1-unstable-2023-10-21";
   rev = "95785196f26e92d82cf4445654ba84e4a9671c57";
 
-  sha256 = "sha256-s1sdXSrLu6uSOmjprbUa4cFsE2Vj7JX5i75e4vRnlvg=";
+  hash = "sha256-s1sdXSrLu6uSOmjprbUa4cFsE2Vj7JX5i75e4vRnlvg=";
 
   isUnstable = true;
+  tests = [
+    nixosTests.zfs.unstable
+  ];
 }

pkgs/top-level/all-packages.nix

@@ -28986,6 +28986,9 @@ with pkgs;
 
   zenmonitor = callPackage ../os-specific/linux/zenmonitor { };
 
+  zfs_2_1 = callPackage ../os-specific/linux/zfs/2_1.nix {
+    configFile = "user";
+  };
   zfsStable = callPackage ../os-specific/linux/zfs/stable.nix {
     configFile = "user";
   };

pkgs/top-level/linux-kernels.nix

@@ -546,6 +546,10 @@ in {
 
     zenpower = callPackage ../os-specific/linux/zenpower { };
 
+    zfs_2_1 = callPackage ../os-specific/linux/zfs/2_1.nix {
+      configFile = "kernel";
+      inherit pkgs kernel;
+    };
     zfsStable = callPackage ../os-specific/linux/zfs/stable.nix {
       configFile = "kernel";
       inherit pkgs kernel;