Merge pull request #160075 from ConnorBaker/spark_3_2_1

spark: init 3.2.1 and test on aarch64-linux

nixos/doc/manual/from_md/release-notes/rl-2205.section.xml

@@ -1456,6 +1456,46 @@
           desktop environments as needed.
         </para>
       </listitem>
+      <listitem>
+        <para>
+          The <literal>hadoop</literal> package has added support for
+          <literal>aarch64-linux</literal> and
+          <literal>aarch64-darwin</literal> as of 3.3.1
+          (<link xlink:href="https://github.com/NixOS/nixpkgs/pull/158613">#158613</link>).
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The <literal>R</literal> package now builds again on
+          <literal>aarch64-darwin</literal>
+          (<link xlink:href="https://github.com/NixOS/nixpkgs/pull/158992">#158992</link>).
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The <literal>spark3</literal> package has been updated from
+          3.1.2 to 3.2.1
+          (<link xlink:href="https://github.com/NixOS/nixpkgs/pull/160075">#160075</link>):
+        </para>
+        <itemizedlist spacing="compact">
+          <listitem>
+            <para>
+              Testing has been enabled for
+              <literal>aarch64-linux</literal> in addition to
+              <literal>x86_64-linux</literal>.
+            </para>
+          </listitem>
+          <listitem>
+            <para>
+              The <literal>spark3</literal> package is now usable on
+              <literal>aarch64-darwin</literal> as a result of
+              <link xlink:href="https://github.com/NixOS/nixpkgs/pull/158613">#158613</link>
+              and
+              <link xlink:href="https://github.com/NixOS/nixpkgs/pull/158992">#158992</link>.
+            </para>
+          </listitem>
+        </itemizedlist>
+      </listitem>
     </itemizedlist>
   </section>
 </section>

nixos/doc/manual/release-notes/rl-2205.section.md

@@ -537,4 +537,13 @@ In addition to numerous new and upgraded packages, this release has the followin
 
 - The polkit service, available at `security.polkit.enable`, is now disabled by default. It will automatically be enabled through services and desktop environments as needed.
 
+- The `hadoop` package has added support for `aarch64-linux` and `aarch64-darwin` as of 3.3.1 ([#158613](https://github.com/NixOS/nixpkgs/pull/158613)).
+
+- The `R` package now builds again on `aarch64-darwin` ([#158992](https://github.com/NixOS/nixpkgs/pull/158992)).
+
+- The `spark3` package has been updated from 3.1.2 to 3.2.1 ([#160075](https://github.com/NixOS/nixpkgs/pull/160075)):
+
+  - Testing has been enabled for `aarch64-linux` in addition to `x86_64-linux`.
+  - The `spark3` package is now usable on `aarch64-darwin` as a result of [#158613](https://github.com/NixOS/nixpkgs/pull/158613) and [#158992](https://github.com/NixOS/nixpkgs/pull/158992).
+
 <!-- To avoid merge conflicts, consider adding your item at an arbitrary place in the list instead. -->

nixos/tests/all-tests.nix

@@ -485,7 +485,7 @@ in
   sonarr = handleTest ./sonarr.nix {};
   sourcehut = handleTest ./sourcehut.nix {};
   spacecookie = handleTest ./spacecookie.nix {};
-  spark = handleTestOn ["x86_64-linux"] ./spark {};
+  spark = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./spark {};
   sslh = handleTest ./sslh.nix {};
   sssd = handleTestOn ["x86_64-linux"] ./sssd.nix {};
   sssd-ldap = handleTestOn ["x86_64-linux"] ./sssd-ldap.nix {};

pkgs/applications/networking/cluster/spark/default.nix

@@ -1,13 +1,26 @@
-{ lib, stdenv, fetchzip, makeWrapper, jdk8, python3Packages, extraPythonPackages ? [], coreutils, hadoop
+{ lib
-, RSupport? true, R
+, stdenv
+, fetchzip
+, makeWrapper
+, jdk8
+, python3Packages
+, extraPythonPackages ? [ ]
+, coreutils
+, hadoop
+, RSupport ? true
+, R
 }:
 
 with lib;
 
 let
-  spark = { pname, version, src }:
+  spark = { pname, version, sha256 }:
     stdenv.mkDerivation rec {
-      inherit pname version src;
+      inherit pname version;
+      src = fetchzip {
+        url = "mirror://apache/spark/${pname}-${version}/${pname}-${version}-bin-without-hadoop.tgz";
+        sha256 = sha256;
+      };
       nativeBuildInputs = [ makeWrapper ];
       buildInputs = [ jdk8 python3Packages.python ]
         ++ extraPythonPackages
@@ -45,31 +58,29 @@ let
       '';
 
       meta = {
-        description      = "Apache Spark is a fast and general engine for large-scale data processing";
+        description = "Apache Spark is a fast and general engine for large-scale data processing";
-        homepage         = "https://spark.apache.org/";
+        homepage = "https://spark.apache.org/";
-        license          = lib.licenses.asl20;
+        license = lib.licenses.asl20;
-        platforms        = lib.platforms.all;
+        platforms = lib.platforms.all;
-        maintainers      = with maintainers; [ thoughtpolice offline kamilchm illustris ];
+        maintainers = with maintainers; [ thoughtpolice offline kamilchm illustris ];
         repositories.git = "git://git.apache.org/spark.git";
       };
     };
-in {
+in
-  spark3 = spark rec {
+{
+  spark_3_2 = spark rec {
+    pname = "spark";
+    version = "3.2.1";
+    sha256 = "0kxdqczwmj6pray0h8h1qhygni9m82jzznw5fbv9hrxrkq1v182d";
+  };
+  spark_3_1 = spark rec {
     pname = "spark";
     version = "3.1.2";
-
+    sha256 = "1bgh2y6jm7wqy6yc40rx68xkki31i3jiri2yixb1bm0i9pvsj9yf";
-    src = fetchzip {
-      url    = "mirror://apache/spark/${pname}-${version}/${pname}-${version}-bin-without-hadoop.tgz";
-      sha256 = "1bgh2y6jm7wqy6yc40rx68xkki31i3jiri2yixb1bm0i9pvsj9yf";
-    };
   };
-  spark2 = spark rec {
+  spark_2_4 = spark rec {
     pname = "spark";
     version = "2.4.8";
-
+    sha256 = "1mkyq0gz9fiav25vr0dba5ivp0wh0mh7kswwnx8pvsmb6wbwyfxv";
-    src = fetchzip {
-      url    = "mirror://apache/spark/${pname}-${version}/${pname}-${version}-bin-without-hadoop.tgz";
-      sha256 = "1mkyq0gz9fiav25vr0dba5ivp0wh0mh7kswwnx8pvsmb6wbwyfxv";
-    };
   };
 }

pkgs/top-level/all-packages.nix

@@ -14163,8 +14163,11 @@ with pkgs;
   self = pkgsi686Linux.callPackage ../development/interpreters/self { };
 
   inherit (callPackages ../applications/networking/cluster/spark { })
-    spark3
+    spark_3_2
-    spark2;
+    spark_3_1
+    spark_2_4;
+  spark3 = spark_3_2;
+  spark2 = spark_2_4;
   spark = spark3;
 
   sparkleshare = callPackage ../applications/version-management/sparkleshare { };