diff --git a/pkgs/development/python-modules/tensorflow-estimator/default.nix b/pkgs/development/python-modules/tensorflow-estimator/default.nix index dd3b4641beaf..5ad6d0ab6e55 100644 --- a/pkgs/development/python-modules/tensorflow-estimator/default.nix +++ b/pkgs/development/python-modules/tensorflow-estimator/default.nix @@ -6,13 +6,13 @@ buildPythonPackage rec { pname = "tensorflow-estimator"; - version = "2.4.0"; + version = "2.7.0"; format = "wheel"; src = fetchPypi { pname = "tensorflow_estimator"; inherit version format; - sha256 = "1w0pkcslm6934qqd6m5gxyjdlnb4pbl47k6s99wsh6dyvvr7nysv"; + hash = "sha256-MltaIkhkN5JCt7dsaYfKVEI5voJXnTPmjsfCvaV6vJ0="; }; propagatedBuildInputs = [ mock numpy absl-py ]; diff --git a/pkgs/development/python-modules/tensorflow/bin.nix b/pkgs/development/python-modules/tensorflow/bin.nix index c14a90e0953e..c57147c8ff7f 100644 --- a/pkgs/development/python-modules/tensorflow/bin.nix +++ b/pkgs/development/python-modules/tensorflow/bin.nix @@ -42,15 +42,11 @@ assert ! (stdenv.isDarwin && cudaSupport); let packages = import ./binary-hashes.nix; - metadataPatch = ./relax-dependencies-metadata.patch; - patch = ./relax-dependencies.patch; in buildPythonPackage { pname = "tensorflow" + lib.optionalString cudaSupport "-gpu"; inherit (packages) version; format = "wheel"; - disabled = pythonAtLeast "3.9"; - src = let pyVerNoDot = lib.strings.stringAsChars (x: if x == "." then "" else x) python.pythonVersion; platform = if stdenv.isDarwin then "mac" else "linux"; @@ -93,13 +89,18 @@ in buildPythonPackage { pushd dist wheel unpack --dest unpacked ./*.whl + rm ./*.whl ( cd unpacked/tensorflow* - # relax too strict versions in setup.py - patch -p 1 < ${patch} - cd *.dist-info - # relax too strict versions in *.dist-info/METADATA - patch -p 3 < ${metadataPatch} + # Adjust dependency requirements: + # - Relax gast version requirement that doesn't match what we have packaged + # - The purpose of python3Packages.libclang is not clear at the moment and we don't have it packaged yet + # - keras and tensorlow-io-gcs-filesystem will be considered as optional for now. + sed -i *.dist-info/METADATA \ + -e "s/Requires-Dist: gast.*/Requires-Dist: gast/" \ + -e "/Requires-Dist: libclang/d" \ + -e "/Requires-Dist: keras/d" \ + -e "/Requires-Dist: tensorflow-io-gcs-filesystem/d" ) wheel pack ./unpacked/tensorflow* diff --git a/pkgs/development/python-modules/tensorflow/binary-hashes.nix b/pkgs/development/python-modules/tensorflow/binary-hashes.nix index 4485e2b61451..4b2f9d528407 100644 --- a/pkgs/development/python-modules/tensorflow/binary-hashes.nix +++ b/pkgs/development/python-modules/tensorflow/binary-hashes.nix @@ -1,39 +1,39 @@ { -version = "2.4.0"; -linux_py_36_cpu = { - url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.4.0-cp36-cp36m-manylinux2010_x86_64.whl"; - sha256 = "0pn0cjf50q0xsv6k0vihrz22kr392anznvdhxv80gk52c6lcgmsc"; -}; +version = "2.7.0"; linux_py_37_cpu = { - url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.4.0-cp37-cp37m-manylinux2010_x86_64.whl"; - sha256 = "0mdd83c0invqfy58qmpa3hk4yml5ic7wlwggyd5wpikadlv8vq89"; + url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.7.0-cp37-cp37m-manylinux2010_x86_64.whl"; + sha256 = "1irw4ri1fbxbj251cnjbr12gxsyj2k30asxjfxjpxhc9b0cchhx0"; }; linux_py_38_cpu = { - url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.4.0-cp38-cp38-manylinux2010_x86_64.whl"; - sha256 = "1mm1yz9aj3v6fxfpxh7wy37rvsncr0b5y6glqlcxmhr6mqfp8k6d"; + url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.7.0-cp38-cp38-manylinux2010_x86_64.whl"; + sha256 = "11a1rimr998lvvjdg03rqr8h2h7x6bbd4gyv74biwgaxmjczvjdj"; }; -linux_py_36_gpu = { - url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-2.4.0-cp36-cp36m-manylinux2010_x86_64.whl"; - sha256 = "113iygiq2kmj97g0glhcqng6rhl3rrj1iqw5xj1d5hla8xjy8cfv"; +linux_py_39_cpu = { + url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.7.0-cp39-cp39-manylinux2010_x86_64.whl"; + sha256 = "0x1zzmgq7hl2k98p8870h0yhaik39nc1ckvxirjx8simpj8ghsbp"; }; linux_py_37_gpu = { - url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-2.4.0-cp37-cp37m-manylinux2010_x86_64.whl"; - sha256 = "19ap8xx2j5nbmnqv5rzf1ryfvw2fbs6bm0fxjqrvhc3jxys6yqqs"; + url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-2.7.0-cp37-cp37m-manylinux2010_x86_64.whl"; + sha256 = "1x5yyxzkly9gdyhcfq4x7w7sqqxh6cw5qnfwbvzj5349xwrm63wi"; }; linux_py_38_gpu = { - url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-2.4.0-cp38-cp38-manylinux2010_x86_64.whl"; - sha256 = "0ly3cinzj6j3b547sw8bd3p774khn3b14cgrj7nvfrz668d3f89x"; + url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-2.7.0-cp38-cp38-manylinux2010_x86_64.whl"; + sha256 = "0prxa8yc6lbq62vnh8cw6dhcrnkdjih62kbg6pbw4gmgcbf6v6fg"; }; -mac_py_36_cpu = { - url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-2.4.0-cp36-cp36m-macosx_10_14_x86_64.whl"; - sha256 = "1b5ld1wj48l1i5s3vk8db5m578zdg4xfl0m1lc8w5lx1vi4cwsjp"; +linux_py_39_gpu = { + url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-2.7.0-cp39-cp39-manylinux2010_x86_64.whl"; + sha256 = "0jjv8vm20d0airml68bxhassp4yg09rcz2a39aa511cz8mavj0l6"; }; mac_py_37_cpu = { - url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-2.4.0-cp37-cp37m-macosx_10_14_x86_64.whl"; - sha256 = "1mgfyjgcwvx5jzawrpfnbch5sqw7kpnzp35rfxj22kwdsl28r47r"; + url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-2.7.0-cp37-cp37m-macosx_10_11_x86_64.whl"; + sha256 = "1gr95ck0h4wzhdacjfhkzxdw4zglm85fc54swqhyzzv1f5n9vw8j"; }; mac_py_38_cpu = { - url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-2.4.0-cp38-cp38-macosx_10_14_x86_64.whl"; - sha256 = "034qh0nk786wxzma58179g086x479c3pd9vi8v8p26grs6f2fm0p"; + url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-2.7.0-cp38-cp38-macosx_10_11_x86_64.whl"; + sha256 = "1h3qlvlrwyvxynh5flvyqk7grbvyqc21ljqhlvds436hpn2kqrgb"; +}; +mac_py_39_cpu = { + url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-2.7.0-cp39-cp39-macosx_10_11_x86_64.whl"; + sha256 = "0pyj1p2yik6ysdqn8n6cz1nqni0s9xx51ifmvdx4dl3hrzp86jz2"; }; } diff --git a/pkgs/development/python-modules/tensorflow/default.nix b/pkgs/development/python-modules/tensorflow/default.nix index e088dec77994..2f8af0692342 100644 --- a/pkgs/development/python-modules/tensorflow/default.nix +++ b/pkgs/development/python-modules/tensorflow/default.nix @@ -4,7 +4,7 @@ , buildPythonPackage, pythonOlder, python # Python libraries , numpy, tensorflow-tensorboard_2, absl-py -, setuptools, wheel, keras-preprocessing, google-pasta +, setuptools, wheel, Keras, keras-preprocessing, google-pasta , opt-einsum, astunparse, h5py , termcolor, grpcio, six, wrapt, protobuf, tensorflow-estimator_2 , dill, flatbuffers-python, tblib, typing-extensions @@ -28,7 +28,7 @@ , avx2Support ? stdenv.hostPlatform.avx2Support , fmaSupport ? stdenv.hostPlatform.fmaSupport # Darwin deps -, Foundation, Security +, Foundation, Security, cctools, llvmPackages_11 }: assert cudaSupport -> cudatoolkit != null @@ -72,7 +72,7 @@ let tfFeature = x: if x then "1" else "0"; - version = "2.4.2"; + version = "2.7.0"; variant = if cudaSupport then "-gpu" else ""; pname = "tensorflow${variant}"; @@ -102,7 +102,82 @@ let wrapt ]); - bazel-build = buildBazelPackage { + rules_cc_darwin_patched = stdenv.mkDerivation { + name = "rules_cc-${pname}-${version}"; + + src = _bazel-build.deps; + + prePatch = "pushd rules_cc"; + patches = [ + # https://github.com/bazelbuild/rules_cc/issues/122 + (fetchpatch { + name = "tensorflow-rules_cc-libtool-path.patch"; + url = "https://github.com/bazelbuild/rules_cc/commit/8c427ab30bf213630dc3bce9d2e9a0e29d1787db.diff"; + sha256 = "sha256-C4v6HY5+jm0ACUZ58gBPVejCYCZfuzYKlHZ0m2qDHCk="; + }) + + # https://github.com/bazelbuild/rules_cc/pull/124 + (fetchpatch { + name = "tensorflow-rules_cc-install_name_tool-path.patch"; + url = "https://github.com/bazelbuild/rules_cc/commit/156497dc89100db8a3f57b23c63724759d431d05.diff"; + sha256 = "sha256-NES1KeQmMiUJQVoV6dS4YGRxxkZEjOpFSCyOq9HZYO0="; + }) + ]; + postPatch = "popd"; + + dontConfigure = true; + dontBuild = true; + + installPhase = '' + runHook preInstall + + mv rules_cc/ "$out" + + runHook postInstall + ''; + }; + llvm-raw_darwin_patched = stdenv.mkDerivation { + name = "llvm-raw-${pname}-${version}"; + + src = _bazel-build.deps; + + prePatch = "pushd llvm-raw"; + patches = [ + # Fix a vendored config.h that requires the 10.13 SDK + ./llvm_bazel_fix_macos_10_12_sdk.patch + ]; + postPatch = '' + touch {BUILD,WORKSPACE} + popd + ''; + + dontConfigure = true; + dontBuild = true; + + installPhase = '' + runHook preInstall + + mv llvm-raw/ "$out" + + runHook postInstall + ''; + }; + bazel-build = if stdenv.isDarwin then _bazel-build.overrideAttrs (prev: { + bazelBuildFlags = prev.bazelBuildFlags ++ [ + "--override_repository=rules_cc=${rules_cc_darwin_patched}" + "--override_repository=llvm-raw=${llvm-raw_darwin_patched}" + ]; + preBuild = '' + export AR="${cctools}/bin/libtool" + ''; + }) else _bazel-build; + + _bazel-build = (buildBazelPackage.override (lib.optionalAttrs stdenv.isDarwin { + # clang 7 fails to emit a symbol for + # __ZN4llvm11SmallPtrSetIPKNS_10AllocaInstELj8EED1Ev in any of the + # translation units, so the build fails at link time + stdenv = llvmPackages_11.stdenv; + })) { name = "${pname}-${version}"; bazel = bazel_3; @@ -110,22 +185,9 @@ let owner = "tensorflow"; repo = "tensorflow"; rev = "v${version}"; - sha256 = "07a2y05hixch1bjag5pzw3p1m7bdj3bq4gdvmsfk2xraz49b1pi8"; + sha256 = "sha256-n7jRDPeXsyq4pEWSWmOCas4c8VsArIKlCuwvSU/Ro/c="; }; - patches = [ - # included from 2.6.0 onwards - (fetchpatch { - name = "fix-numpy-1.20-notimplementederror.patch"; - url = "https://github.com/tensorflow/tensorflow/commit/b258941525f496763d4277045b6513c815720e3a.patch"; - sha256 = "19f9bzrcfsynk11s2hqvscin5c65zf7r6g3nb10jnimw79vafiry"; - }) - # Relax too strict Python packages versions dependencies. - ./relax-dependencies.patch - # Add missing `io_bazel_rules_docker` dependency. - ./workspace.patch - ]; - # On update, it can be useful to steal the changes from gentoo # https://gitweb.gentoo.org/repo/gentoo.git/tree/sci-libs/tensorflow @@ -206,7 +268,6 @@ let "opt_einsum_archive" "org_sqlite" "pasta" - "pcre" "png" "pybind11" "six_archive" @@ -295,9 +356,12 @@ let fetchAttrs = { # cudaSupport causes fetch of ncclArchive, resulting in different hashes sha256 = if cudaSupport then - "10m6qj3kchgxfgb6qh59vc51knm9r9pkng8bf90h00dnggvv8234" + "sha256-GIBs1BAUuefwlavu7dr9rFb4n1A3uwnvvCAvsBnSSqQ=" else - "04a98yrp09nd0p17k0jbzkgjppxs0yma7m5zkfrwgvr4g0w71v68"; + if stdenv.isDarwin then + "sha256-156eOnnjk+wzIiGLd6k/+SAgm4AyImsV/qBsHFlxe+k=" + else + "sha256-Fj/wWapsre55VctJ1k1kcYKAn3uDCMPN5rVX8y76ypM="; }; buildAttrs = { @@ -355,6 +419,18 @@ in buildPythonPackage { src = bazel-build.python; + # Adjust dependency requirements: + # - Relax gast version requirement that doesn't match what we have packaged + # - The purpose of python3Packages.libclang is not clear at the moment and we don't have it packaged yet + # - keras and tensorlow-io-gcs-filesystem will be considered as optional for now. + postPatch = '' + sed -i setup.py \ + -e "s/'gast[^']*',/'gast',/" \ + -e "/'libclang[^']*',/d" \ + -e "/'keras[^']*',/d" \ + -e "/'tensorflow-io-gcs-filesystem[^']*',/d" + ''; + # Upstream has a pip hack that results in bin/tensorboard being in both tensorflow # and the propagated input tensorflow-tensorboard, which causes environment collisions. # Another possibility would be to have tensorboard only in the buildInputs @@ -402,6 +478,7 @@ in buildPythonPackage { # Actual tests are slow and impure. # TODO try to run them anyway # TODO better test (files in tensorflow/tools/ci_build/builds/*test) + checkInputs = [ Keras ]; checkPhase = '' ${python.interpreter} <=3.9.2) --Requires-Dist: six (~=1.15.0) --Requires-Dist: termcolor (~=1.1.0) --Requires-Dist: typing-extensions (~=3.7.4) --Requires-Dist: wheel (~=0.35) --Requires-Dist: wrapt (~=1.12.1) --Requires-Dist: gast (==0.3.3) --Requires-Dist: tensorboard (~=2.4) --Requires-Dist: tensorflow-estimator (<2.5.0,>=2.4.0rc0) --Requires-Dist: grpcio (~=1.32.0) -+Requires-Dist: absl-py -+Requires-Dist: astunparse -+Requires-Dist: flatbuffers -+Requires-Dist: google-pasta -+Requires-Dist: h5py -+Requires-Dist: keras-preprocessing -+Requires-Dist: numpy -+Requires-Dist: opt-einsum -+Requires-Dist: protobuf -+Requires-Dist: six -+Requires-Dist: termcolor -+Requires-Dist: typing-extensions -+Requires-Dist: wheel -+Requires-Dist: wrapt -+Requires-Dist: gast -+Requires-Dist: tensorboard -+Requires-Dist: tensorflow-estimator -+Requires-Dist: grpcio - - [![Python](https://img.shields.io/pypi/pyversions/tensorflow.svg?style=plastic)](https://badge.fury.io/py/tensorflow) - [![PyPI](https://badge.fury.io/py/tensorflow.svg)](https://badge.fury.io/py/tensorflow) \ No newline at end of file diff --git a/pkgs/development/python-modules/tensorflow/relax-dependencies.patch b/pkgs/development/python-modules/tensorflow/relax-dependencies.patch deleted file mode 100644 index b26c1e95a645..000000000000 --- a/pkgs/development/python-modules/tensorflow/relax-dependencies.patch +++ /dev/null @@ -1,51 +0,0 @@ -diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py -index 65133afdafe..8ef6364ff7e 100644 ---- a/tensorflow/tools/pip_package/setup.py -+++ b/tensorflow/tools/pip_package/setup.py -@@ -75,23 +75,23 @@ if '--project_name' in sys.argv: - # comment the versioning scheme. - # NOTE: Please add test only packages to `TEST_PACKAGES` below. - REQUIRED_PACKAGES = [ -- 'absl-py ~= 0.10', -- 'astunparse ~= 1.6.3', -- 'flatbuffers ~= 1.12.0', -- 'google_pasta ~= 0.2', -- 'h5py ~= 2.10.0', -- 'keras_preprocessing ~= 1.1.2', -- 'numpy ~= 1.19.2', -- 'opt_einsum ~= 3.3.0', -+ 'absl-py >= 0.10', -+ 'astunparse >= 1.6.3', -+ 'flatbuffers >= 1.12.0', -+ 'google_pasta >= 0.2', -+ 'h5py >= 2.10.0', -+ 'keras_preprocessing >= 1.1.2', -+ 'numpy >= 1.19.1', -+ 'opt_einsum >= 3.3.0', - 'protobuf >= 3.9.2', -- 'six ~= 1.15.0', -- 'termcolor ~= 1.1.0', -- 'typing_extensions ~= 3.7.4', -- 'wheel ~= 0.35', -- 'wrapt ~= 1.12.1', -+ 'six >= 1.15.0', -+ 'termcolor >= 1.1.0', -+ 'typing_extensions >= 3.7.4', -+ 'wheel >= 0.34.2', -+ 'wrapt >= 1.12.1', - # These packages needs to be pinned exactly as newer versions are - # incompatible with the rest of the ecosystem -- 'gast == 0.3.3', -+ 'gast >= 0.3.3', - # TensorFlow ecosystem packages that TF exposes API for - # These need to be in sync with the existing TF version - # They are updated during the release process -@@ -118,7 +118,7 @@ if 'tf_nightly' in project_name: - # BoringSSL support. - # See https://github.com/tensorflow/tensorflow/issues/17882. - if sys.byteorder == 'little': -- REQUIRED_PACKAGES.append('grpcio ~= 1.32.0') -+ REQUIRED_PACKAGES.append('grpcio >= 1.31.0') - - - # Packages which are only needed for testing code. diff --git a/pkgs/development/python-modules/tensorflow/workspace.patch b/pkgs/development/python-modules/tensorflow/workspace.patch deleted file mode 100644 index 15a527815ec3..000000000000 --- a/pkgs/development/python-modules/tensorflow/workspace.patch +++ /dev/null @@ -1,18 +0,0 @@ -diff --git a/WORKSPACE b/WORKSPACE -index 9db1d9b80eb..c46f13f4ca4 100644 ---- a/WORKSPACE -+++ b/WORKSPACE -@@ -12,6 +12,13 @@ http_archive( - ], - ) - -+http_archive( -+ name = "io_bazel_rules_docker", -+ sha256 = "1698624e878b0607052ae6131aa216d45ebb63871ec497f26c67455b34119c80", -+ strip_prefix = "rules_docker-0.15.0", -+ urls = ["https://github.com/bazelbuild/rules_docker/releases/download/v0.15.0/rules_docker-v0.15.0.tar.gz"], -+) -+ - # Load tf_repositories() before loading dependencies for other repository so - # that dependencies like com_google_protobuf won't be overridden. - load("//tensorflow:workspace.bzl", "tf_repositories") diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 120b16eb5695..a827f8a9394d 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -9230,6 +9230,7 @@ in { tensorflow-bin = self.tensorflow-bin_2; tensorflow-build_2 = callPackage ../development/python-modules/tensorflow { + inherit (pkgs.darwin) cctools; cudaSupport = pkgs.config.cudaSupport or false; cudatoolkit = pkgs.cudatoolkit_11_0; cudnn = pkgs.cudnn_cudatoolkit_11_0;