From bc260c31f12e3f21542cfe3828663225995e9554 Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Sun, 6 Jun 2021 21:18:56 +0000 Subject: [PATCH 1/7] buildBazelPackage: don't fail if the marker file doesn't exist This was the intent of my previous change, but it didn't quite work because the pipeline still exits false even if the file doesn't exist. Oops. --- pkgs/build-support/build-bazel-package/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/build-support/build-bazel-package/default.nix b/pkgs/build-support/build-bazel-package/default.nix index 988298ac72bf..502be398eaa3 100644 --- a/pkgs/build-support/build-bazel-package/default.nix +++ b/pkgs/build-support/build-bazel-package/default.nix @@ -126,7 +126,7 @@ in stdenv.mkDerivation (fBuildAttrs // { find $bazelOut/external -maxdepth 1 -type l | while read symlink; do name="$(basename "$symlink")" rm "$symlink" - test -f "$bazelOut/external/@$name.marker" && rm "$bazelOut/external/@$name.marker" + test -f "$bazelOut/external/@$name.marker" && rm "$bazelOut/external/@$name.marker" || true done # Patching symlinks to remove build directory reference From cfd38a4006163277e2de9cf0810c23ba5d6c7e1d Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Sun, 6 Jun 2021 21:28:19 +0000 Subject: [PATCH 2/7] buildBazelPackage: set fetchConfigured default to true --- nixos/doc/manual/from_md/release-notes/rl-2111.section.xml | 7 +++++++ nixos/doc/manual/release-notes/rl-2111.section.md | 3 +++ pkgs/build-support/build-bazel-package/default.nix | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml index 7ac120b693f0..adda1025d2f3 100644 --- a/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml +++ b/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml @@ -56,6 +56,13 @@ this version for the entire lifecycle of the 21.11 release. + + + Those making use of buildBazelPackage will + need to regenerate the fetch hashes (preferred), or set + fetchConfigured = false;. + +
diff --git a/nixos/doc/manual/release-notes/rl-2111.section.md b/nixos/doc/manual/release-notes/rl-2111.section.md index 577d75929352..b7a4e13f75e6 100644 --- a/nixos/doc/manual/release-notes/rl-2111.section.md +++ b/nixos/doc/manual/release-notes/rl-2111.section.md @@ -24,4 +24,7 @@ In addition to numerous new and upgraded packages, this release has the followin * PHP 7.3 is no longer supported due to upstream not supporting this version for the entire lifecycle of the 21.11 release. +* Those making use of `buildBazelPackage` will need to regenerate the fetch + hashes (preferred), or set `fetchConfigured = false;`. + ## Other Notable Changes diff --git a/pkgs/build-support/build-bazel-package/default.nix b/pkgs/build-support/build-bazel-package/default.nix index 502be398eaa3..198b9c3f617f 100644 --- a/pkgs/build-support/build-bazel-package/default.nix +++ b/pkgs/build-support/build-bazel-package/default.nix @@ -35,7 +35,7 @@ args@{ # required for the build as configured, rather than fetching all the dependencies # which may not work in some situations (e.g. Java code which ends up relying on # Debian-specific /usr/share/java paths, but doesn't in the configured build). -, fetchConfigured ? false +, fetchConfigured ? true # Don’t add Bazel --copt and --linkopt from NIX_CFLAGS_COMPILE / # NIX_LDFLAGS. This is necessary when using a custom toolchain which From 5b30dd9b1c6a62cd10f3854117cf61ed2c2d841a Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Sun, 6 Jun 2021 21:29:00 +0000 Subject: [PATCH 3/7] gvisor: 2019-11-14 -> 20210518.0 This also now seems to depend on protoc, but we substitute out the version of protoc so that we don't download the binary artifacts, and instead compile protoc from support. TBD: using the Nixpkgs built version of protoc instead of building from source. --- .../virtualization/gvisor/default.nix | 35 +++++++++++++++---- pkgs/top-level/all-packages.nix | 2 +- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/pkgs/applications/virtualization/gvisor/default.nix b/pkgs/applications/virtualization/gvisor/default.nix index e6b83286c00d..4cd043d4eb6d 100644 --- a/pkgs/applications/virtualization/gvisor/default.nix +++ b/pkgs/applications/virtualization/gvisor/default.nix @@ -1,6 +1,8 @@ { lib , buildBazelPackage , fetchFromGitHub +, callPackage +, bash , cacert , git , glibcLocales @@ -9,6 +11,7 @@ , iptables , makeWrapper , procps +, protobuf , python3 }: @@ -16,9 +19,12 @@ let preBuild = '' patchShebangs . + substituteInPlace tools/defs.bzl \ + --replace "#!/bin/bash" "#!${bash}/bin/bash" + # Tell rules_go to use the Go binary found in the PATH sed -E -i \ - -e 's|go_version\s*=\s*"[^"]+",|go_version = "host",|g' \ + -e 's|go_version\s*=\s*"[^"]+"|go_version = "host"|g' \ WORKSPACE # The gazelle Go tooling needs CA certs @@ -31,20 +37,37 @@ let export GOPATH= ''; + # Patch the protoc alias so that it always builds from source. + rulesProto = fetchFromGitHub { + owner = "bazelbuild"; + repo = "rules_proto"; + rev = "f7a30f6f80006b591fa7c437fe5a951eb10bcbcf"; + sha256 = "10bcw0ir0skk7h33lmqm38n9w4nfs24mwajnngkbs6jb5wsvkqv8"; + extraPostFetch = '' + sed -i 's|name = "protoc"|name = "_protoc_original"|' $out/proto/private/BUILD.release + cat <>$out/proto/private/BUILD.release + alias(name = "protoc", actual = "@com_github_protocolbuffers_protobuf//:protoc", visibility = ["//visibility:public"]) + EOF + ''; + }; + in buildBazelPackage rec { name = "gvisor-${version}"; - version = "2019-11-14"; + version = "20210518.0"; src = fetchFromGitHub { owner = "google"; repo = "gvisor"; - rev = "release-20191114.0"; - sha256 = "0kyixjjlws9iz2r2srgpdd4rrq94vpxkmh2rmmzxd9mcqy2i9bg1"; + rev = "release-${version}"; + sha256 = "15a6mlclnyfc9mx3bjksnnf4vla0xh0rv9kxdp34la4gw3c4hksn"; }; nativeBuildInputs = [ git glibcLocales go makeWrapper python3 ]; bazelTarget = "//runsc:runsc"; + bazelFlags = [ + "--override_repository=rules_proto=${rulesProto}" + ]; # gvisor uses the Starlark implementation of rules_cc, not the built-in one, # so we shouldn't delete it from our dependencies. @@ -76,14 +99,14 @@ in buildBazelPackage rec { rm -f "$bazelOut"/java.log "$bazelOut"/java.log.* ''; - sha256 = "0fhmlq0d2317gwhma2mz1anb69j4chybk90j71j88wpgw1hxbk34"; + sha256 = "13pahppm431m198v5bffrzq5iw8m79riplbfqp0afh384ln669hb"; }; buildAttrs = { inherit preBuild; installPhase = '' - install -Dm755 bazel-bin/runsc/*_pure_stripped/runsc $out/bin/runsc + install -Dm755 bazel-out/*/bin/runsc/runsc_/runsc $out/bin/runsc # Needed for the 'runsc do' subcomand wrapProgram $out/bin/runsc \ diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index c537b0545a37..72a53a99cb49 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -24160,7 +24160,7 @@ in gv = callPackage ../applications/misc/gv { }; gvisor = callPackage ../applications/virtualization/gvisor { - go = go_1_14; + go = go_1_16; }; gvisor-containerd-shim = callPackage ../applications/virtualization/gvisor/containerd-shim.nix { }; From 00e54055e1e7d3fbaacced361a8999a6e2ac8104 Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Mon, 7 Jun 2021 01:18:01 +0000 Subject: [PATCH 4/7] envoy: 1.16.2 -> 1.17.3 --- pkgs/servers/http/envoy/default.nix | 33 +++++++++++++++++------------ pkgs/top-level/all-packages.nix | 5 ++++- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/pkgs/servers/http/envoy/default.nix b/pkgs/servers/http/envoy/default.nix index 57c0e22d8aff..d26782560a47 100644 --- a/pkgs/servers/http/envoy/default.nix +++ b/pkgs/servers/http/envoy/default.nix @@ -3,7 +3,9 @@ , fetchFromGitHub , stdenv , cmake +, gn , go +, jdk , ninja , python3 , nixosTests @@ -15,8 +17,8 @@ let # However, the version string is more useful for end-users. # These are contained in a attrset of their own to make it obvious that # people should update both. - version = "1.16.2"; - commit = "e98e41a8e168af7acae8079fc0cd68155f699aa3"; + version = "1.17.3"; + commit = "46bf743b97d0d3f01ff437b2f10cc0bd9cdfe6e4"; }; in buildBazelPackage rec { @@ -26,7 +28,7 @@ buildBazelPackage rec { owner = "envoyproxy"; repo = "envoy"; rev = srcVer.commit; - hash = "sha256-aWVMRKFCZzf9/96NRPCP4jiW38DJhXyi0gEqW7uIpnQ="; + hash = "sha256:09zzr4h3zjsb2rkxrvlazpx0jy33yn9j65ilxiqbvv0ckaralqfc"; extraPostFetch = '' chmod -R +w $out @@ -36,29 +38,29 @@ buildBazelPackage rec { ''; }; - patches = [ - # Quiche needs to be updated to compile under newer GCC. - # This is a manual backport of https://github.com/envoyproxy/envoy/pull/13949. - ./0001-quiche-update-QUICHE-tar-13949.patch - - # upb needs to be updated to compile under newer GCC. - # This is a manual backport of https://github.com/protocolbuffers/upb/commit/9bd23dab4240b015321a53c45b3c9e4847fbf020. - ./0002-Add-upb-patch-to-make-it-compile-under-GCC10.patch - ]; postPatch = '' sed -i 's,#!/usr/bin/env python3,#!${python3}/bin/python,' bazel/foreign_cc/luajit.patch + sed -i '/javabase=/d' .bazelrc + # Patch paths to build tools, and disable gold because it just segfaults. + substituteInPlace bazel/external/wee8.genrule_cmd \ + --replace '"''$$gn"' '"''$$(command -v gn)"' \ + --replace '"''$$ninja"' '"''$$(command -v ninja)"' \ + --replace '"''$$WEE8_BUILD_ARGS"' '"''$$WEE8_BUILD_ARGS use_gold=false"' ''; nativeBuildInputs = [ cmake python3 + gn go + jdk ninja ]; fetchAttrs = { - sha256 = "0q72c2zrl5vc8afkhkwyalb2h0mxn3133d4b9z4gag0p95wbwgc0"; + sha256 = "sha256:1cy2b73x8jzczq9z9c1kl7zrg5iasvsakb50zxn4mswpmajkbj5h"; dontUseCmakeConfigure = true; + dontUseGnConfigure = true; preInstall = '' # Strip out the path to the build location (by deleting the comment line). find $bazelOut/external -name requirements.bzl | while read requirements; do @@ -78,10 +80,14 @@ buildBazelPackage rec { >$bazelOut/external/config_validation_pip3/PyYAML-5.3.1-cp38-cp38-linux_x86_64.whl >$bazelOut/external/protodoc_pip3/PyYAML-5.3.1-cp38-cp38-linux_x86_64.whl >$bazelOut/external/thrift_pip3/thrift-0.13.0-cp38-cp38-linux_x86_64.whl + + # Remove Unix timestamps from go cache. + rm -rf $bazelOut/external/bazel_gazelle_go_repository_cache/{gocache,pkg/mod/cache,pkg/sumdb} ''; }; buildAttrs = { dontUseCmakeConfigure = true; + dontUseGnConfigure = true; dontUseNinjaInstall = true; preConfigure = '' sed -i 's,#!/usr/bin/env bash,#!${stdenv.shell},' $bazelOut/external/rules_foreign_cc/tools/build_defs/framework.bzl @@ -98,7 +104,6 @@ buildBazelPackage rec { ''; }; - fetchConfigured = true; removeRulesCC = false; removeLocalConfigCc = true; removeLocal = false; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 72a53a99cb49..d238973c554f 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -19098,7 +19098,10 @@ in engelsystem = callPackage ../servers/web-apps/engelsystem { php = php74; }; - envoy = callPackage ../servers/http/envoy { }; + envoy = callPackage ../servers/http/envoy { + go = go_1_15; + jdk = openjdk11; + }; etcd = callPackage ../servers/etcd { }; etcd_3_4 = callPackage ../servers/etcd/3.4.nix { }; From 137fcc6cf048ec6a8ea8766af7636e9bdf32d063 Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Mon, 7 Jun 2021 01:18:16 +0000 Subject: [PATCH 5/7] pomerium: 0.13.3 -> 0.14.4 --- pkgs/servers/http/pomerium/default.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/servers/http/pomerium/default.nix b/pkgs/servers/http/pomerium/default.nix index 0605a12eca47..036e44ca621e 100644 --- a/pkgs/servers/http/pomerium/default.nix +++ b/pkgs/servers/http/pomerium/default.nix @@ -11,15 +11,15 @@ let in buildGoModule rec { pname = "pomerium"; - version = "0.13.3"; + version = "0.14.4"; src = fetchFromGitHub { owner = "pomerium"; repo = "pomerium"; rev = "v${version}"; - hash = "sha256-g0w1aIHvf2rJANvGWHeUxdnyCDsvy/PQ9Kp8nDdT/0w="; + hash = "sha256:097csr8f43cn0iq030ajvvpwnwcfmjxyyk6pcisdy937axlrzska"; }; - vendorSha256 = "sha256-grihU85OcGyf9/KKrv87xZonX5r+Z1oHQTf84Ya61fg="; + vendorSha256 = "sha256:0n45xvwjiqyh41dsm4z1rnkgkycf5wfmacm804hqnd5rz7xk2shf"; subPackages = [ "cmd/pomerium" "cmd/pomerium-cli" From f18734f8b89764d21c7eb7c62014565e444cf981 Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Mon, 7 Jun 2021 01:52:23 +0000 Subject: [PATCH 6/7] python3Packages.tensorflow: update hashes for fetchConfigured --- pkgs/development/python-modules/tensorflow/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/tensorflow/default.nix b/pkgs/development/python-modules/tensorflow/default.nix index 522fd7301271..bbdecc8bdcdf 100644 --- a/pkgs/development/python-modules/tensorflow/default.nix +++ b/pkgs/development/python-modules/tensorflow/default.nix @@ -288,9 +288,9 @@ let fetchAttrs = { # cudaSupport causes fetch of ncclArchive, resulting in different hashes sha256 = if cudaSupport then - "1i7z2a7bc2q1vn1h9nx1xc6g1r1cby2xvbcs20fj9h6c2fgaw9j4" + "10m6qj3kchgxfgb6qh59vc51knm9r9pkng8bf90h00dnggvv8234" else - "0s8q5rxq8abr50c5jpwv96ncfc0k8jw7w70ri8viqy031g9v9v45"; + "04a98yrp09nd0p17k0jbzkgjppxs0yma7m5zkfrwgvr4g0w71v68"; }; buildAttrs = { From 3d6fd379920bc9b906b60da3a17ad0a4d2d142c7 Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Mon, 7 Jun 2021 01:54:29 +0000 Subject: [PATCH 7/7] bazel-watcher: update hashes for fetchConfigured --- pkgs/development/tools/bazel-watcher/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/development/tools/bazel-watcher/default.nix b/pkgs/development/tools/bazel-watcher/default.nix index 657704a653c7..e10a455c0906 100644 --- a/pkgs/development/tools/bazel-watcher/default.nix +++ b/pkgs/development/tools/bazel-watcher/default.nix @@ -56,7 +56,7 @@ buildBazelPackage rec { sed -e '/^FILE:@bazel_gazelle_go_repository_tools.*/d' -i $bazelOut/external/\@*.marker ''; - sha256 = "0rwwjjj6zaj4hdcbsbp0di53xn6203r2vgpddhdrp8iph9ab60cg"; + sha256 = "1j175z3d4fbi4pl35py7yjq7ywrvwin6id131jv32hx0ck4g1m46"; }; buildAttrs = {