hadoop: add passthrough tests

nixos/tests/all-tests.nix

@@ -189,9 +189,9 @@ in
   grocy = handleTest ./grocy.nix {};
   grub = handleTest ./grub.nix {};
   gvisor = handleTest ./gvisor.nix {};
-  hadoop.all = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hadoop/hadoop.nix {};
-  hadoop.hdfs = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hadoop/hdfs.nix {};
-  hadoop.yarn = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hadoop/yarn.nix {};
+  hadoop = import ./hadoop { inherit handleTestOn; package=pkgs.hadoop; };
+  hadoop_3_2 = import ./hadoop { inherit handleTestOn; package=pkgs.hadoop_3_2; };
+  hadoop2 = import ./hadoop { inherit handleTestOn; package=pkgs.hadoop2; };
   haka = handleTest ./haka.nix {};
   haproxy = handleTest ./haproxy.nix {};
   hardened = handleTest ./hardened.nix {};

nixos/tests/hadoop/default.nix

@@ -0,0 +1,7 @@
+{ handleTestOn, package, ... }:
+
+{
+  all = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hadoop.nix { inherit package; };
+  hdfs = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hdfs.nix { inherit package; };
+  yarn = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./yarn.nix { inherit package; };
+}

nixos/tests/hadoop/hadoop.nix

@@ -1,10 +1,11 @@
 # This test is very comprehensive. It tests whether all hadoop services work well with each other.
 # Run this when updating the Hadoop package or making significant changes to the hadoop module.
 # For a more basic test, see hdfs.nix and yarn.nix
-import ../make-test-python.nix ({pkgs, ...}: {
+import ../make-test-python.nix ({ package, ... }: {
+  name = "hadoop-combined";
 
-  nodes = let
-    package = pkgs.hadoop;
+  nodes =
+    let
       coreSite = {
         "fs.defaultFS" = "hdfs://ns1";
       };
@@ -45,14 +46,15 @@ import ../make-test-python.nix ({pkgs, ...}: {
         "yarn.resourcemanager.webapp.address.rm1" = "rm1:8088";
         "yarn.resourcemanager.webapp.address.rm2" = "rm2:8088";
       };
-  in {
+    in
+    {
       zk1 = { ... }: {
         services.zookeeper.enable = true;
         networking.firewall.allowedTCPPorts = [ 2181 ];
       };
 
       # HDFS cluster
-    nn1 = {pkgs, options, ...}: {
+      nn1 = { ... }: {
         services.hadoop = {
           inherit package coreSite hdfsSite;
           hdfs.namenode = {
@@ -62,7 +64,7 @@ import ../make-test-python.nix ({pkgs, ...}: {
           hdfs.zkfc.enable = true;
         };
       };
-    nn2 = {pkgs, options, ...}: {
+      nn2 = { ... }: {
         services.hadoop = {
           inherit package coreSite hdfsSite;
           hdfs.namenode = {
@@ -73,7 +75,7 @@ import ../make-test-python.nix ({pkgs, ...}: {
         };
       };
 
-    jn1 = {pkgs, options, ...}: {
+      jn1 = { ... }: {
         services.hadoop = {
           inherit package coreSite hdfsSite;
           hdfs.journalnode = {
@@ -82,7 +84,7 @@ import ../make-test-python.nix ({pkgs, ...}: {
           };
         };
       };
-    jn2 = {pkgs, options, ...}: {
+      jn2 = { ... }: {
         services.hadoop = {
           inherit package coreSite hdfsSite;
           hdfs.journalnode = {
@@ -91,7 +93,7 @@ import ../make-test-python.nix ({pkgs, ...}: {
           };
         };
       };
-    jn3 = {pkgs, options, ...}: {
+      jn3 = { ... }: {
         services.hadoop = {
           inherit package coreSite hdfsSite;
           hdfs.journalnode = {
@@ -101,7 +103,7 @@ import ../make-test-python.nix ({pkgs, ...}: {
         };
       };
 
-    dn1 = {pkgs, options, ...}: {
+      dn1 = { ... }: {
         services.hadoop = {
           inherit package coreSite hdfsSite;
           hdfs.datanode = {
@@ -112,7 +114,7 @@ import ../make-test-python.nix ({pkgs, ...}: {
       };
 
       # YARN cluster
-    rm1 = {pkgs, options, ...}: {
+      rm1 = { options, ... }: {
         services.hadoop = {
           inherit package coreSite hdfsSite;
           yarnSite = options.services.hadoop.yarnSite.default // yarnSiteHA;
@@ -122,7 +124,7 @@ import ../make-test-python.nix ({pkgs, ...}: {
           };
         };
       };
-    rm2 = {pkgs, options, ...}: {
+      rm2 = { options, ... }: {
         services.hadoop = {
           inherit package coreSite hdfsSite;
           yarnSite = options.services.hadoop.yarnSite.default // yarnSiteHA;
@@ -132,7 +134,7 @@ import ../make-test-python.nix ({pkgs, ...}: {
           };
         };
       };
-    nm1 = {pkgs, options, ...}: {
+      nm1 = { options, ... }: {
         virtualisation.memorySize = 2048;
         services.hadoop = {
           inherit package coreSite hdfsSite;

nixos/tests/hadoop/hdfs.nix

@@ -1,9 +1,11 @@
 # Test a minimal HDFS cluster with no HA
-import ../make-test-python.nix ({...}: {
+import ../make-test-python.nix ({ package, ... }: {
+  name = "hadoop-hdfs";
+
   nodes = {
-    namenode = {pkgs, ...}: {
+    namenode = { pkgs, ... }: {
       services.hadoop = {
-        package = pkgs.hadoop;
+        inherit package;
         hdfs = {
           namenode = {
             enable = true;
@@ -22,9 +24,9 @@ import ../make-test-python.nix ({...}: {
         };
       };
     };
-    datanode = {pkgs, ...}: {
+    datanode = { pkgs, ... }: {
       services.hadoop = {
-        package = pkgs.hadoop;
+        inherit package;
         hdfs.datanode = {
           enable = true;
           openFirewall = true;

nixos/tests/hadoop/yarn.nix

@@ -1,28 +1,33 @@
 # This only tests if YARN is able to start its services
-import ../make-test-python.nix ({...}: {
+import ../make-test-python.nix ({ package, ... }: {
+  name = "hadoop-yarn";
+
   nodes = {
-    resourcemanager = {pkgs, ...}: {
-      services.hadoop.package = pkgs.hadoop;
-      services.hadoop.yarn.resourcemanager = {
+    resourcemanager = { ... }: {
+      services.hadoop = {
+        inherit package;
+        yarn.resourcemanager = {
           enable = true;
           openFirewall = true;
         };
-      services.hadoop.yarnSite = {
+        yarnSite = {
           "yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler";
         };
       };
-    nodemanager = {pkgs, ...}: {
-      services.hadoop.package = pkgs.hadoop;
-      services.hadoop.yarn.nodemanager = {
+    };
+    nodemanager = { ... }: {
+      services.hadoop = {
+        inherit package;
+        yarn.nodemanager = {
           enable = true;
           openFirewall = true;
         };
-      services.hadoop.yarnSite = {
+        yarnSite = {
           "yarn.resourcemanager.hostname" = "resourcemanager";
           "yarn.nodemanager.log-dirs" = "/tmp/userlogs";
         };
       };
-
+    };
   };
 
   testScript = ''

pkgs/applications/networking/cluster/hadoop/default.nix

@@ -15,6 +15,8 @@
 , zlib
 , zstd
 , openssl
+, openssl
+, nixosTests
 }:
 
 with lib;
@@ -22,7 +24,7 @@ with lib;
 assert elem stdenv.system [ "x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin" ];
 
 let
-  common = { pname, version, untarDir ? "${pname}-${version}", sha256, jdk, openssl ? null, nativeLibs ? [ ], libPatches ? "" }:
+  common = { pname, version, untarDir ? "${pname}-${version}", sha256, jdk, openssl ? null, nativeLibs ? [ ], libPatches ? "", tests }:
     stdenv.mkDerivation rec {
       inherit pname version jdk libPatches untarDir openssl;
       src = fetchurl {
@@ -49,6 +51,8 @@ let
         done
       '' + libPatches;
 
+      passthru = { inherit tests; };
+
       meta = {
         homepage = "https://hadoop.apache.org/";
         description = "Framework for distributed processing of large data sets across clusters of computers";
@@ -73,9 +77,7 @@ in
 {
   # Different version of hadoop support different java runtime versions
   # https://cwiki.apache.org/confluence/display/HADOOP/Hadoop+Java+Versions
-  hadoop_3_3 =
-    common
-      (rec {
+  hadoop_3_3 = common rec {
     pname = "hadoop";
     version = "3.3.1";
     untarDir = "${pname}-${version}";
@@ -96,7 +98,8 @@ in
       ln -s ${getLib bzip2}/lib/libbz2.so.1 $out/lib/${untarDir}/lib/native/
     '' + optionalString stdenv.isLinux "patchelf --add-rpath ${jdk.home}/lib/server $out/lib/${untarDir}/lib/native/libnativetask.so.1.0.0";
     jdk = jdk11_headless;
-      });
+    tests = nixosTests.hadoop;
+  };
   hadoop_3_2 = common rec {
     pname = "hadoop";
     version = "3.2.2";
@@ -104,11 +107,15 @@ in
     jdk = jdk8_headless;
     # not using native libs because of broken openssl_1_0_2 dependency
     # can be manually overriden
+    # Disable tests involving HDFS till the module adds support for hadoop_3_2
+    tests = nixosTests.hadoop_3_2 // { all = null; hdfs = null; };
   };
   hadoop2 = common rec {
     pname = "hadoop";
     version = "2.10.1";
     sha256.x86_64-linux = "1w31x4bk9f2swnx8qxx0cgwfg8vbpm6cy5lvfnbbpl3rsjhmyg97";
     jdk = jdk8_headless;
+    # Disable tests involving HDFS till the module adds support for hadoop2
+    tests = nixosTests.hadoop2 // { all = null; hdfs = null; };
   };
 }