Merge pull request #160075 from ConnorBaker/spark_3_2_1
spark: init 3.2.1 and test on aarch64-linux
This commit is contained in:
commit
47f2ee3d55
5 changed files with 88 additions and 25 deletions
nixos
pkgs
|
@ -1456,6 +1456,46 @@
|
|||
desktop environments as needed.
|
||||
</para>
|
||||
</listitem>
|
||||
<listitem>
|
||||
<para>
|
||||
The <literal>hadoop</literal> package has added support for
|
||||
<literal>aarch64-linux</literal> and
|
||||
<literal>aarch64-darwin</literal> as of 3.3.1
|
||||
(<link xlink:href="https://github.com/NixOS/nixpkgs/pull/158613">#158613</link>).
|
||||
</para>
|
||||
</listitem>
|
||||
<listitem>
|
||||
<para>
|
||||
The <literal>R</literal> package now builds again on
|
||||
<literal>aarch64-darwin</literal>
|
||||
(<link xlink:href="https://github.com/NixOS/nixpkgs/pull/158992">#158992</link>).
|
||||
</para>
|
||||
</listitem>
|
||||
<listitem>
|
||||
<para>
|
||||
The <literal>spark3</literal> package has been updated from
|
||||
3.1.2 to 3.2.1
|
||||
(<link xlink:href="https://github.com/NixOS/nixpkgs/pull/160075">#160075</link>):
|
||||
</para>
|
||||
<itemizedlist spacing="compact">
|
||||
<listitem>
|
||||
<para>
|
||||
Testing has been enabled for
|
||||
<literal>aarch64-linux</literal> in addition to
|
||||
<literal>x86_64-linux</literal>.
|
||||
</para>
|
||||
</listitem>
|
||||
<listitem>
|
||||
<para>
|
||||
The <literal>spark3</literal> package is now usable on
|
||||
<literal>aarch64-darwin</literal> as a result of
|
||||
<link xlink:href="https://github.com/NixOS/nixpkgs/pull/158613">#158613</link>
|
||||
and
|
||||
<link xlink:href="https://github.com/NixOS/nixpkgs/pull/158992">#158992</link>.
|
||||
</para>
|
||||
</listitem>
|
||||
</itemizedlist>
|
||||
</listitem>
|
||||
</itemizedlist>
|
||||
</section>
|
||||
</section>
|
||||
|
|
|
@ -537,4 +537,13 @@ In addition to numerous new and upgraded packages, this release has the followin
|
|||
|
||||
- The polkit service, available at `security.polkit.enable`, is now disabled by default. It will automatically be enabled through services and desktop environments as needed.
|
||||
|
||||
- The `hadoop` package has added support for `aarch64-linux` and `aarch64-darwin` as of 3.3.1 ([#158613](https://github.com/NixOS/nixpkgs/pull/158613)).
|
||||
|
||||
- The `R` package now builds again on `aarch64-darwin` ([#158992](https://github.com/NixOS/nixpkgs/pull/158992)).
|
||||
|
||||
- The `spark3` package has been updated from 3.1.2 to 3.2.1 ([#160075](https://github.com/NixOS/nixpkgs/pull/160075)):
|
||||
|
||||
- Testing has been enabled for `aarch64-linux` in addition to `x86_64-linux`.
|
||||
- The `spark3` package is now usable on `aarch64-darwin` as a result of [#158613](https://github.com/NixOS/nixpkgs/pull/158613) and [#158992](https://github.com/NixOS/nixpkgs/pull/158992).
|
||||
|
||||
<!-- To avoid merge conflicts, consider adding your item at an arbitrary place in the list instead. -->
|
||||
|
|
|
@ -485,7 +485,7 @@ in
|
|||
sonarr = handleTest ./sonarr.nix {};
|
||||
sourcehut = handleTest ./sourcehut.nix {};
|
||||
spacecookie = handleTest ./spacecookie.nix {};
|
||||
spark = handleTestOn ["x86_64-linux"] ./spark {};
|
||||
spark = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./spark {};
|
||||
sslh = handleTest ./sslh.nix {};
|
||||
sssd = handleTestOn ["x86_64-linux"] ./sssd.nix {};
|
||||
sssd-ldap = handleTestOn ["x86_64-linux"] ./sssd-ldap.nix {};
|
||||
|
|
|
@ -1,13 +1,26 @@
|
|||
{ lib, stdenv, fetchzip, makeWrapper, jdk8, python3Packages, extraPythonPackages ? [], coreutils, hadoop
|
||||
, RSupport? true, R
|
||||
{ lib
|
||||
, stdenv
|
||||
, fetchzip
|
||||
, makeWrapper
|
||||
, jdk8
|
||||
, python3Packages
|
||||
, extraPythonPackages ? [ ]
|
||||
, coreutils
|
||||
, hadoop
|
||||
, RSupport ? true
|
||||
, R
|
||||
}:
|
||||
|
||||
with lib;
|
||||
|
||||
let
|
||||
spark = { pname, version, src }:
|
||||
spark = { pname, version, sha256 }:
|
||||
stdenv.mkDerivation rec {
|
||||
inherit pname version src;
|
||||
inherit pname version;
|
||||
src = fetchzip {
|
||||
url = "mirror://apache/spark/${pname}-${version}/${pname}-${version}-bin-without-hadoop.tgz";
|
||||
sha256 = sha256;
|
||||
};
|
||||
nativeBuildInputs = [ makeWrapper ];
|
||||
buildInputs = [ jdk8 python3Packages.python ]
|
||||
++ extraPythonPackages
|
||||
|
@ -45,31 +58,29 @@ let
|
|||
'';
|
||||
|
||||
meta = {
|
||||
description = "Apache Spark is a fast and general engine for large-scale data processing";
|
||||
homepage = "https://spark.apache.org/";
|
||||
license = lib.licenses.asl20;
|
||||
platforms = lib.platforms.all;
|
||||
maintainers = with maintainers; [ thoughtpolice offline kamilchm illustris ];
|
||||
description = "Apache Spark is a fast and general engine for large-scale data processing";
|
||||
homepage = "https://spark.apache.org/";
|
||||
license = lib.licenses.asl20;
|
||||
platforms = lib.platforms.all;
|
||||
maintainers = with maintainers; [ thoughtpolice offline kamilchm illustris ];
|
||||
repositories.git = "git://git.apache.org/spark.git";
|
||||
};
|
||||
};
|
||||
in {
|
||||
spark3 = spark rec {
|
||||
in
|
||||
{
|
||||
spark_3_2 = spark rec {
|
||||
pname = "spark";
|
||||
version = "3.2.1";
|
||||
sha256 = "0kxdqczwmj6pray0h8h1qhygni9m82jzznw5fbv9hrxrkq1v182d";
|
||||
};
|
||||
spark_3_1 = spark rec {
|
||||
pname = "spark";
|
||||
version = "3.1.2";
|
||||
|
||||
src = fetchzip {
|
||||
url = "mirror://apache/spark/${pname}-${version}/${pname}-${version}-bin-without-hadoop.tgz";
|
||||
sha256 = "1bgh2y6jm7wqy6yc40rx68xkki31i3jiri2yixb1bm0i9pvsj9yf";
|
||||
};
|
||||
sha256 = "1bgh2y6jm7wqy6yc40rx68xkki31i3jiri2yixb1bm0i9pvsj9yf";
|
||||
};
|
||||
spark2 = spark rec {
|
||||
spark_2_4 = spark rec {
|
||||
pname = "spark";
|
||||
version = "2.4.8";
|
||||
|
||||
src = fetchzip {
|
||||
url = "mirror://apache/spark/${pname}-${version}/${pname}-${version}-bin-without-hadoop.tgz";
|
||||
sha256 = "1mkyq0gz9fiav25vr0dba5ivp0wh0mh7kswwnx8pvsmb6wbwyfxv";
|
||||
};
|
||||
sha256 = "1mkyq0gz9fiav25vr0dba5ivp0wh0mh7kswwnx8pvsmb6wbwyfxv";
|
||||
};
|
||||
}
|
||||
|
|
|
@ -14163,8 +14163,11 @@ with pkgs;
|
|||
self = pkgsi686Linux.callPackage ../development/interpreters/self { };
|
||||
|
||||
inherit (callPackages ../applications/networking/cluster/spark { })
|
||||
spark3
|
||||
spark2;
|
||||
spark_3_2
|
||||
spark_3_1
|
||||
spark_2_4;
|
||||
spark3 = spark_3_2;
|
||||
spark2 = spark_2_4;
|
||||
spark = spark3;
|
||||
|
||||
sparkleshare = callPackage ../applications/version-management/sparkleshare { };
|
||||
|
|
Loading…
Reference in a new issue