cafkafk/nixpkgs
cafkafk/nixpkgs
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

hadoop: add passthrough tests

Browse source
This commit is contained in:
illustris 2022-01-08 18:38:34 +05:30
parent c82d48913f
commit 799dc66cf1
6 changed files with 199 additions and 176 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
nixos/tests/all-tests.nix
View file

@ -189,9 +189,9 @@ in
grocy = handleTest ./grocy.nix {}; grocy = handleTest ./grocy.nix {};
grub = handleTest ./grub.nix {}; grub = handleTest ./grub.nix {};
gvisor = handleTest ./gvisor.nix {}; gvisor = handleTest ./gvisor.nix {};
hadoop.all = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hadoop/hadoop.nix {}; hadoop = import ./hadoop { inherit handleTestOn; package=pkgs.hadoop; };
hadoop.hdfs = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hadoop/hdfs.nix {}; hadoop_3_2 = import ./hadoop { inherit handleTestOn; package=pkgs.hadoop_3_2; };
hadoop.yarn = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hadoop/yarn.nix {}; hadoop2 = import ./hadoop { inherit handleTestOn; package=pkgs.hadoop2; };
haka = handleTest ./haka.nix {}; haka = handleTest ./haka.nix {};
haproxy = handleTest ./haproxy.nix {}; haproxy = handleTest ./haproxy.nix {};
hardened = handleTest ./hardened.nix {}; hardened = handleTest ./hardened.nix {};

7
nixos/tests/hadoop/default.nix Normal file
View file

@ -0,0 +1,7 @@
{ handleTestOn, package, ... }:
{
all = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hadoop.nix { inherit package; };
hdfs = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hdfs.nix { inherit package; };
yarn = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./yarn.nix { inherit package; };
}

28
nixos/tests/hadoop/hadoop.nix
View file

@ -1,10 +1,11 @@
# This test is very comprehensive. It tests whether all hadoop services work well with each other. # This test is very comprehensive. It tests whether all hadoop services work well with each other.
# Run this when updating the Hadoop package or making significant changes to the hadoop module. # Run this when updating the Hadoop package or making significant changes to the hadoop module.
# For a more basic test, see hdfs.nix and yarn.nix # For a more basic test, see hdfs.nix and yarn.nix
import ../make-test-python.nix ({pkgs, ...}: { import ../make-test-python.nix ({ package, ... }: {
name = "hadoop-combined";
nodes = let nodes =
package = pkgs.hadoop; let
coreSite = { coreSite = {
"fs.defaultFS" = "hdfs://ns1"; "fs.defaultFS" = "hdfs://ns1";
}; };
@ -45,14 +46,15 @@ import ../make-test-python.nix ({pkgs, ...}: {
"yarn.resourcemanager.webapp.address.rm1" = "rm1:8088"; "yarn.resourcemanager.webapp.address.rm1" = "rm1:8088";
"yarn.resourcemanager.webapp.address.rm2" = "rm2:8088"; "yarn.resourcemanager.webapp.address.rm2" = "rm2:8088";
}; };
in { in
{
zk1 = { ... }: { zk1 = { ... }: {
services.zookeeper.enable = true; services.zookeeper.enable = true;
networking.firewall.allowedTCPPorts = [ 2181 ]; networking.firewall.allowedTCPPorts = [ 2181 ];
}; };
# HDFS cluster # HDFS cluster
nn1 = {pkgs, options, ...}: { nn1 = { ... }: {
services.hadoop = { services.hadoop = {
inherit package coreSite hdfsSite; inherit package coreSite hdfsSite;
hdfs.namenode = { hdfs.namenode = {
@ -62,7 +64,7 @@ import ../make-test-python.nix ({pkgs, ...}: {
hdfs.zkfc.enable = true; hdfs.zkfc.enable = true;
}; };
}; };
nn2 = {pkgs, options, ...}: { nn2 = { ... }: {
services.hadoop = { services.hadoop = {
inherit package coreSite hdfsSite; inherit package coreSite hdfsSite;
hdfs.namenode = { hdfs.namenode = {
@ -73,7 +75,7 @@ import ../make-test-python.nix ({pkgs, ...}: {
}; };
}; };
jn1 = {pkgs, options, ...}: { jn1 = { ... }: {
services.hadoop = { services.hadoop = {
inherit package coreSite hdfsSite; inherit package coreSite hdfsSite;
hdfs.journalnode = { hdfs.journalnode = {
@ -82,7 +84,7 @@ import ../make-test-python.nix ({pkgs, ...}: {
}; };
}; };
}; };
jn2 = {pkgs, options, ...}: { jn2 = { ... }: {
services.hadoop = { services.hadoop = {
inherit package coreSite hdfsSite; inherit package coreSite hdfsSite;
hdfs.journalnode = { hdfs.journalnode = {
@ -91,7 +93,7 @@ import ../make-test-python.nix ({pkgs, ...}: {
}; };
}; };
}; };
jn3 = {pkgs, options, ...}: { jn3 = { ... }: {
services.hadoop = { services.hadoop = {
inherit package coreSite hdfsSite; inherit package coreSite hdfsSite;
hdfs.journalnode = { hdfs.journalnode = {
@ -101,7 +103,7 @@ import ../make-test-python.nix ({pkgs, ...}: {
}; };
}; };
dn1 = {pkgs, options, ...}: { dn1 = { ... }: {
services.hadoop = { services.hadoop = {
inherit package coreSite hdfsSite; inherit package coreSite hdfsSite;
hdfs.datanode = { hdfs.datanode = {
@ -112,7 +114,7 @@ import ../make-test-python.nix ({pkgs, ...}: {
}; };
# YARN cluster # YARN cluster
rm1 = {pkgs, options, ...}: { rm1 = { options, ... }: {
services.hadoop = { services.hadoop = {
inherit package coreSite hdfsSite; inherit package coreSite hdfsSite;
yarnSite = options.services.hadoop.yarnSite.default // yarnSiteHA; yarnSite = options.services.hadoop.yarnSite.default // yarnSiteHA;
@ -122,7 +124,7 @@ import ../make-test-python.nix ({pkgs, ...}: {
}; };
}; };
}; };
rm2 = {pkgs, options, ...}: { rm2 = { options, ... }: {
services.hadoop = { services.hadoop = {
inherit package coreSite hdfsSite; inherit package coreSite hdfsSite;
yarnSite = options.services.hadoop.yarnSite.default // yarnSiteHA; yarnSite = options.services.hadoop.yarnSite.default // yarnSiteHA;
@ -132,7 +134,7 @@ import ../make-test-python.nix ({pkgs, ...}: {
}; };
}; };
}; };
nm1 = {pkgs, options, ...}: { nm1 = { options, ... }: {
virtualisation.memorySize = 2048; virtualisation.memorySize = 2048;
services.hadoop = { services.hadoop = {
inherit package coreSite hdfsSite; inherit package coreSite hdfsSite;

8
nixos/tests/hadoop/hdfs.nix
View file

@ -1,9 +1,11 @@
# Test a minimal HDFS cluster with no HA # Test a minimal HDFS cluster with no HA
import ../make-test-python.nix ({...}: { import ../make-test-python.nix ({ package, ... }: {
name = "hadoop-hdfs";
nodes = { nodes = {
namenode = { pkgs, ... }: { namenode = { pkgs, ... }: {
services.hadoop = { services.hadoop = {
package = pkgs.hadoop; inherit package;
hdfs = { hdfs = {
namenode = { namenode = {
enable = true; enable = true;
@ -24,7 +26,7 @@ import ../make-test-python.nix ({...}: {
}; };
datanode = { pkgs, ... }: { datanode = { pkgs, ... }: {
services.hadoop = { services.hadoop = {
package = pkgs.hadoop; inherit package;
hdfs.datanode = { hdfs.datanode = {
enable = true; enable = true;
openFirewall = true; openFirewall = true;

25
nixos/tests/hadoop/yarn.nix
View file

@ -1,28 +1,33 @@
# This only tests if YARN is able to start its services # This only tests if YARN is able to start its services
import ../make-test-python.nix ({...}: { import ../make-test-python.nix ({ package, ... }: {
name = "hadoop-yarn";
nodes = { nodes = {
resourcemanager = {pkgs, ...}: { resourcemanager = { ... }: {
services.hadoop.package = pkgs.hadoop; services.hadoop = {
services.hadoop.yarn.resourcemanager = { inherit package;
yarn.resourcemanager = {
enable = true; enable = true;
openFirewall = true; openFirewall = true;
}; };
services.hadoop.yarnSite = { yarnSite = {
"yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler"; "yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler";
}; };
}; };
nodemanager = {pkgs, ...}: { };
services.hadoop.package = pkgs.hadoop; nodemanager = { ... }: {
services.hadoop.yarn.nodemanager = { services.hadoop = {
inherit package;
yarn.nodemanager = {
enable = true; enable = true;
openFirewall = true; openFirewall = true;
}; };
services.hadoop.yarnSite = { yarnSite = {
"yarn.resourcemanager.hostname" = "resourcemanager"; "yarn.resourcemanager.hostname" = "resourcemanager";
"yarn.nodemanager.log-dirs" = "/tmp/userlogs"; "yarn.nodemanager.log-dirs" = "/tmp/userlogs";
}; };
}; };
};
}; };
testScript = '' testScript = ''

17
pkgs/applications/networking/cluster/hadoop/default.nix
View file

@ -15,6 +15,8 @@
, zlib , zlib
, zstd , zstd
, openssl , openssl
, openssl
, nixosTests
}: }:
with lib; with lib;
@ -22,7 +24,7 @@ with lib;
assert elem stdenv.system [ "x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin" ]; assert elem stdenv.system [ "x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin" ];
let let
common = { pname, version, untarDir ? "${pname}-${version}", sha256, jdk, openssl ? null, nativeLibs ? [ ], libPatches ? "" }: common = { pname, version, untarDir ? "${pname}-${version}", sha256, jdk, openssl ? null, nativeLibs ? [ ], libPatches ? "", tests }:
stdenv.mkDerivation rec { stdenv.mkDerivation rec {
inherit pname version jdk libPatches untarDir openssl; inherit pname version jdk libPatches untarDir openssl;
src = fetchurl { src = fetchurl {
@ -49,6 +51,8 @@ let
done done
'' + libPatches; '' + libPatches;
passthru = { inherit tests; };
meta = { meta = {
homepage = "https://hadoop.apache.org/"; homepage = "https://hadoop.apache.org/";
description = "Framework for distributed processing of large data sets across clusters of computers"; description = "Framework for distributed processing of large data sets across clusters of computers";
@ -73,9 +77,7 @@ in
{ {
# Different version of hadoop support different java runtime versions # Different version of hadoop support different java runtime versions
# https://cwiki.apache.org/confluence/display/HADOOP/Hadoop+Java+Versions # https://cwiki.apache.org/confluence/display/HADOOP/Hadoop+Java+Versions
hadoop_3_3 = hadoop_3_3 = common rec {
common
(rec {
pname = "hadoop"; pname = "hadoop";
version = "3.3.1"; version = "3.3.1";
untarDir = "${pname}-${version}"; untarDir = "${pname}-${version}";
@ -96,7 +98,8 @@ in
ln -s ${getLib bzip2}/lib/libbz2.so.1 $out/lib/${untarDir}/lib/native/ ln -s ${getLib bzip2}/lib/libbz2.so.1 $out/lib/${untarDir}/lib/native/
'' + optionalString stdenv.isLinux "patchelf --add-rpath ${jdk.home}/lib/server $out/lib/${untarDir}/lib/native/libnativetask.so.1.0.0"; '' + optionalString stdenv.isLinux "patchelf --add-rpath ${jdk.home}/lib/server $out/lib/${untarDir}/lib/native/libnativetask.so.1.0.0";
jdk = jdk11_headless; jdk = jdk11_headless;
}); tests = nixosTests.hadoop;
};
hadoop_3_2 = common rec { hadoop_3_2 = common rec {
pname = "hadoop"; pname = "hadoop";
version = "3.2.2"; version = "3.2.2";
@ -104,11 +107,15 @@ in
jdk = jdk8_headless; jdk = jdk8_headless;
# not using native libs because of broken openssl_1_0_2 dependency # not using native libs because of broken openssl_1_0_2 dependency
# can be manually overriden # can be manually overriden
# Disable tests involving HDFS till the module adds support for hadoop_3_2
tests = nixosTests.hadoop_3_2 // { all = null; hdfs = null; };
}; };
hadoop2 = common rec { hadoop2 = common rec {
pname = "hadoop"; pname = "hadoop";
version = "2.10.1"; version = "2.10.1";
sha256.x86_64-linux = "1w31x4bk9f2swnx8qxx0cgwfg8vbpm6cy5lvfnbbpl3rsjhmyg97"; sha256.x86_64-linux = "1w31x4bk9f2swnx8qxx0cgwfg8vbpm6cy5lvfnbbpl3rsjhmyg97";
jdk = jdk8_headless; jdk = jdk8_headless;
# Disable tests involving HDFS till the module adds support for hadoop2
tests = nixosTests.hadoop2 // { all = null; hdfs = null; };
}; };
} }