From 7429acdea1c5b8abc60ebada23112ccd2dd6a313 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Fri, 16 Aug 2019 16:21:48 +0200 Subject: [PATCH 1/5] bazel: Test that all shebangs are patched correctly --- .../tools/build-managers/bazel/default.nix | 2 + .../build-managers/bazel/shebang-test.nix | 47 +++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 pkgs/development/tools/build-managers/bazel/shebang-test.nix diff --git a/pkgs/development/tools/build-managers/bazel/default.nix b/pkgs/development/tools/build-managers/bazel/default.nix index e9ce95c16099..352b79f21b6f 100644 --- a/pkgs/development/tools/build-managers/bazel/default.nix +++ b/pkgs/development/tools/build-managers/bazel/default.nix @@ -208,6 +208,7 @@ stdenv.mkDerivation rec { # about why to create a subdir for the workspace. cp -r ${workspaceDir} wd && chmod u+w wd && cd wd + BAZEL_EXTRACTED=${be.install_dir} ${bazelScript} touch $out @@ -223,6 +224,7 @@ stdenv.mkDerivation rec { }; in { + shebang = callPackage ./shebang-test.nix { inherit runLocal bazelTest distDir; }; bashTools = callPackage ./bash-tools-test.nix { inherit runLocal bazelTest distDir; }; cpp = callPackage ./cpp-test.nix { inherit runLocal bazelTest bazel-examples distDir; }; java = callPackage ./java-test.nix { inherit runLocal bazelTest bazel-examples distDir; }; diff --git a/pkgs/development/tools/build-managers/bazel/shebang-test.nix b/pkgs/development/tools/build-managers/bazel/shebang-test.nix new file mode 100644 index 000000000000..98ec9a67c156 --- /dev/null +++ b/pkgs/development/tools/build-managers/bazel/shebang-test.nix @@ -0,0 +1,47 @@ +{ + bazel +, bazelTest +, distDir +, runLocal +, unzip +}: + +# Tests that all shebangs are patched appropriately. +# #!/usr/bin/... should be replaced by Nix store references. +# #!.../bin/env python should be replaced by Nix store reference to the python interpreter. + +let + + workspaceDir = runLocal "our_workspace" {} "mkdir $out"; + + testBazel = bazelTest { + name = "bazel-test-shebangs"; + inherit workspaceDir; + bazelPkg = bazel; + bazelScript = '' + set -ueo pipefail + FAIL= + check_shebangs() { + local dir="$1" + { grep -Re '#!/usr/bin' $dir && FAIL=1; } || true + { grep -Re '#![^[:space:]]*/bin/env python' $dir && FAIL=1; } || true + } + check_shebangs $BAZEL_EXTRACTED + while IFS= read -r -d "" zip; do + unzipped="./$zip/UNPACKED" + mkdir -p "$unzipped" + unzip -qq $zip -d "$unzipped" + check_shebangs "$unzipped" + rm -rf unzipped + done < <(find $BAZEL_EXTRACTED -type f -name '*.zip' -or -name '*.jar' -print0) + if [[ $FAIL = 1 ]]; then + echo "Found files in the bazel distribution with illegal shebangs." >&2 + echo "Replace those by explicit Nix store paths." >&2 + echo "Python scripts should not use \`bin/env python' but the Python interpreter's store path." >&2 + exit 1 + fi + ''; + buildInputs = [ unzip ]; + }; + +in testBazel From d132d47199b8f89486e6f1f514a2938a50e735fa Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Mon, 19 Aug 2019 14:09:59 +0200 Subject: [PATCH 2/5] bazel: shebang-test: pass bazel dir in Nix --- pkgs/development/tools/build-managers/bazel/default.nix | 3 +-- pkgs/development/tools/build-managers/bazel/shebang-test.nix | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkgs/development/tools/build-managers/bazel/default.nix b/pkgs/development/tools/build-managers/bazel/default.nix index 352b79f21b6f..c0ff09d09767 100644 --- a/pkgs/development/tools/build-managers/bazel/default.nix +++ b/pkgs/development/tools/build-managers/bazel/default.nix @@ -208,7 +208,6 @@ stdenv.mkDerivation rec { # about why to create a subdir for the workspace. cp -r ${workspaceDir} wd && chmod u+w wd && cd wd - BAZEL_EXTRACTED=${be.install_dir} ${bazelScript} touch $out @@ -224,7 +223,7 @@ stdenv.mkDerivation rec { }; in { - shebang = callPackage ./shebang-test.nix { inherit runLocal bazelTest distDir; }; + shebang = callPackage ./shebang-test.nix { inherit runLocal extracted bazelTest distDir; }; bashTools = callPackage ./bash-tools-test.nix { inherit runLocal bazelTest distDir; }; cpp = callPackage ./cpp-test.nix { inherit runLocal bazelTest bazel-examples distDir; }; java = callPackage ./java-test.nix { inherit runLocal bazelTest bazel-examples distDir; }; diff --git a/pkgs/development/tools/build-managers/bazel/shebang-test.nix b/pkgs/development/tools/build-managers/bazel/shebang-test.nix index 98ec9a67c156..8610b036f2cc 100644 --- a/pkgs/development/tools/build-managers/bazel/shebang-test.nix +++ b/pkgs/development/tools/build-managers/bazel/shebang-test.nix @@ -2,6 +2,7 @@ bazel , bazelTest , distDir +, extracted , runLocal , unzip }: @@ -26,6 +27,7 @@ let { grep -Re '#!/usr/bin' $dir && FAIL=1; } || true { grep -Re '#![^[:space:]]*/bin/env python' $dir && FAIL=1; } || true } + BAZEL_EXTRACTED=${extracted bazel}/install check_shebangs $BAZEL_EXTRACTED while IFS= read -r -d "" zip; do unzipped="./$zip/UNPACKED" From f6572b4e646ecfe6f836aef993bb0df874f0082d Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Mon, 19 Aug 2019 14:28:23 +0200 Subject: [PATCH 3/5] bazel: patch #!/usr/bin/env bash To point to the custom bash instead of `/nix/store.../bin/env bash`. --- .../tools/build-managers/bazel/default.nix | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pkgs/development/tools/build-managers/bazel/default.nix b/pkgs/development/tools/build-managers/bazel/default.nix index c0ff09d09767..95617241c1b6 100644 --- a/pkgs/development/tools/build-managers/bazel/default.nix +++ b/pkgs/development/tools/build-managers/bazel/default.nix @@ -5,7 +5,7 @@ , lr, xe, zip, unzip, bash, writeCBin, coreutils , which, gawk, gnused, gnutar, gnugrep, gzip, findutils # updater -, python3, writeScript +, python27, python3, writeScript # Apple dependencies , cctools, libcxx, CoreFoundation, CoreServices, Foundation # Allow to independently override the jdks used to build and run respectively @@ -328,10 +328,11 @@ stdenv.mkDerivation rec { ''; genericPatches = '' - # Substitute python's stub shebang to plain python path. (see TODO add pr URL) + # Substitute j2objc and objc wrapper's python shebang to plain python path. # See also `postFixup` where python is added to $out/nix-support - substituteInPlace src/main/java/com/google/devtools/build/lib/bazel/rules/python/python_stub_template.txt \ - --replace "#!/usr/bin/env python" "#!${python3}/bin/python" + substituteInPlace tools/j2objc/j2objc_header_map.py --replace "$!/usr/bin/python2.7" "#!${python27}/bin/python" + substituteInPlace tools/j2objc/j2objc_wrapper.py --replace "$!/usr/bin/python2.7" "#!${python27}/bin/python" + substituteInPlace tools/objc/j2objc_dead_code_pruner.py --replace "$!/usr/bin/python2.7" "#!${python27}/bin/python" # md5sum is part of coreutils sed -i 's|/sbin/md5|md5sum|' \ @@ -343,6 +344,8 @@ stdenv.mkDerivation rec { # Only files containing /bin are taken into account. substituteInPlace "$path" \ --replace /bin/bash ${customBash}/bin/bash \ + --replace "/usr/bin/env bash" ${customBash}/bin/bash \ + --replace "/usr/bin/env python" ${python3}/bin/python \ --replace /usr/bin/env ${coreutils}/bin/env \ --replace /bin/true ${coreutils}/bin/true done From 1987d860070dfda0f6f6f877eb1d7b72fb985aa6 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Mon, 19 Aug 2019 14:29:07 +0200 Subject: [PATCH 4/5] bazel: shebang-test: Test for all `bin/env ...` Fail on any form of `bin/env ...` shebang. --- pkgs/development/tools/build-managers/bazel/shebang-test.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/development/tools/build-managers/bazel/shebang-test.nix b/pkgs/development/tools/build-managers/bazel/shebang-test.nix index 8610b036f2cc..fd94f97a7659 100644 --- a/pkgs/development/tools/build-managers/bazel/shebang-test.nix +++ b/pkgs/development/tools/build-managers/bazel/shebang-test.nix @@ -25,7 +25,7 @@ let check_shebangs() { local dir="$1" { grep -Re '#!/usr/bin' $dir && FAIL=1; } || true - { grep -Re '#![^[:space:]]*/bin/env python' $dir && FAIL=1; } || true + { grep -Re '#![^[:space:]]*/bin/env' $dir && FAIL=1; } || true } BAZEL_EXTRACTED=${extracted bazel}/install check_shebangs $BAZEL_EXTRACTED From 1f3187c1b2ae4549e212118089e2f2daeebc6eb6 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Wed, 21 Aug 2019 14:39:33 +0200 Subject: [PATCH 5/5] bazel: Comment on python versions. --- pkgs/development/tools/build-managers/bazel/default.nix | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkgs/development/tools/build-managers/bazel/default.nix b/pkgs/development/tools/build-managers/bazel/default.nix index 95617241c1b6..d5cbd642e705 100644 --- a/pkgs/development/tools/build-managers/bazel/default.nix +++ b/pkgs/development/tools/build-managers/bazel/default.nix @@ -329,7 +329,8 @@ stdenv.mkDerivation rec { genericPatches = '' # Substitute j2objc and objc wrapper's python shebang to plain python path. - # See also `postFixup` where python is added to $out/nix-support + # These scripts explicitly depend on Python 2.7, hence we use python27. + # See also `postFixup` where python27 is added to $out/nix-support substituteInPlace tools/j2objc/j2objc_header_map.py --replace "$!/usr/bin/python2.7" "#!${python27}/bin/python" substituteInPlace tools/j2objc/j2objc_wrapper.py --replace "$!/usr/bin/python2.7" "#!${python27}/bin/python" substituteInPlace tools/objc/j2objc_dead_code_pruner.py --replace "$!/usr/bin/python2.7" "#!${python27}/bin/python" @@ -342,6 +343,8 @@ stdenv.mkDerivation rec { grep -rlZ /bin src/main/java/com/google/devtools | while IFS="" read -r -d "" path; do # If you add more replacements here, you must change the grep above! # Only files containing /bin are taken into account. + # We default to python3 where possible. See also `postFixup` where + # python3 is added to $out/nix-support substituteInPlace "$path" \ --replace /bin/bash ${customBash}/bin/bash \ --replace "/usr/bin/env bash" ${customBash}/bin/bash \ @@ -521,6 +524,10 @@ stdenv.mkDerivation rec { echo "${customBash} ${defaultShellPath}" >> $out/nix-support/depends # The templates get tar’d up into a .jar, # so nix can’t detect python is needed in the runtime closure + # Some of the scripts explicitly depend on Python 2.7. Otherwise, we + # default to using python3. Therefore, both python27 and python3 are + # runtime dependencies. + echo "${python27}" >> $out/nix-support/depends echo "${python3}" >> $out/nix-support/depends '' + lib.optionalString stdenv.isDarwin '' echo "${cctools}" >> $out/nix-support/depends