Rework repository_cache, lockfile generation, and protobuf tests as a whole

pkgs/development/tools/build-managers/bazel/bazel_7/bazel-repository-cache.nix

@@ -8,9 +8,11 @@
 , requiredDepNamePredicate ? _: true
   # Extra deps in the form of derivations producing a "single file" output path
 , extraDeps ? [ ]
+, canonicalIds ? true
 }:
 let
   modules = builtins.fromJSON (builtins.readFile lockfile);
+  modulesVersion = modules.lockFileVersion;
 
   # a foldl' for json values
   foldlJSON = op: acc: value:
@@ -31,7 +33,10 @@ let
     builtins.seq acc' children;
 
   # remove the "--" prefix, abusing undocumented negative substring length
-  sanitize = builtins.substring 2 (-1);
+  sanitize = str:
+    if modulesVersion < 3
+    then builtins.substring 2 (-1) str
+    else str;
 
   extract_source = f: acc: value:
     # We take any "attributes" object that has a "sha256" field. Every value
@@ -59,6 +64,7 @@ let
           sha256 = hash;
           passthru.sha256 = hash;
           passthru.source_name = name;
+          passthru.urls = urls;
         };
       };
       insert = acc: hash: urls: name:
@@ -76,7 +82,8 @@ let
     if builtins.isAttrs value && value ? attributes
       && (attrs ? sha256 || attrs ? integrity)
       && f attrs.name
-    then accWithNewSource else acc;
+    then accWithNewSource
+    else acc;
 
   requiredSourcePredicate = n: requiredDepNamePredicate (sanitize n);
   requiredDeps = foldlJSON (extract_source requiredSourcePredicate) { } modules;
@@ -89,10 +96,23 @@ let
     (drv: ''
       filename=$(basename "${lib.head drv.urls}")
       echo Bundling $filename ${lib.optionalString (drv?source_name) "from ${drv.source_name}"}
+
+      # 1. --repository_cache format:
+      # 1.a. A file unde a content-hash directory
       hash=$(${rnix-hashes}/bin/rnix-hashes --encoding BASE16 ${drv.sha256} | cut -f 2)
       mkdir -p content_addressable/sha256/$hash
       ln -sfn ${drv} content_addressable/sha256/$hash/file
-      # Expect file name conflicts
+
+      # 1.b. a canonicalId marker based on the download urls
+      # Bazel uses these to avoid reusing a stale hash when the urls have changed.
+      canonicalId="${lib.concatStringsSep " " drv.urls}"
+      canonicalIdHash=$(echo -n "$canonicalId" | sha256sum | cut -d" " -f1)
+      echo -n "$canonicalId" > content_addressable/sha256/$hash/id-$canonicalIdHash
+
+      # 2. --distdir format:
+      # Just a file with the right basename
+      # Mostly to keep old tests happy, and because symlinks cost nothing.
+      # This is brittle because of expected file name conflicts
       ln -sn ${drv} $filename || true
     '')
     (builtins.attrValues requiredDeps ++ extraDeps)

pkgs/development/tools/build-managers/bazel/bazel_7/default.nix

@@ -58,39 +58,32 @@
 }@args:
 
 let
-  version = "7.0.0-pre.20230917.3";
+  version = "7.0.0-pre.20231011.2";
   sourceRoot = ".";
 
   src = fetchurl {
     url = "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-dist.zip";
-    hash = "sha256-sxITaivcJRrQrL+zZtdZohesNgmDtQysIG3BS8SFZd4=";
+    hash = "sha256-3kwNnFPGyRLBKSSzsa+pTNAHowH+fLjtwxiSY8RkbQc=";
   };
 
   # Use builtins.fetchurl to avoid IFD, in particular on hydra
   lockfile = builtins.fetchurl {
     url = "https://raw.githubusercontent.com/bazelbuild/bazel/${version}/MODULE.bazel.lock";
-    sha256 = "0z6mlz8cn03qa40mqbw6j6kd6qyn4vgb3bb1kyidazgldxjhrz6y";
+    sha256 = "sha256:0wdwkfq3mrg4dxvmq2prdsj1i5jyql8jc2wf3wx948cnc10pbfy7";
   };
 
   # Two-in-one format
   distDir = repoCache;
   repoCache = callPackage ./bazel-repository-cache.nix {
     inherit lockfile;
-    # We use the release tarball that already has everything bundled,
+
-    # But we need one extra dep required by our nonprebuilt java toolchains.
+    # We use the release tarball that already has everything bundled so we
+    # should not need any extra external deps. But our nonprebuilt java
+    # toolchains hack needs just one non bundled dep.
     requiredDepNamePredicate = name:
       null != builtins.match "rules_java~.*~toolchains~remote_java_tools" name;
   };
 
-  # Two-in-one format
-  testRepoCache = callPackage ./bazel-repository-cache.nix {
-    inherit lockfile;
-    # We use the release tarball that already has everything bundled,
-    # But we need one extra dep required by our nonprebuilt java toolchains.
-    requiredDepNamePredicate = name:
-      null == builtins.match ".*(macos|osx|linux|win|apple|android).*" name;
-  };
-
   defaultShellUtils =
     # Keep this list conservative. For more exotic tools, prefer to use
     # @rules_nixpkgs to pull in tools from the nix repository. Example:
@@ -379,8 +372,10 @@ stdenv.mkDerivation rec {
           -e "/bazel_build /a\  --experimental_strict_java_deps=off \\\\" \
           -e "/bazel_build /a\  --strict_proto_deps=off \\\\" \
           -e "/bazel_build /a\  --toolchain_resolution_debug='@bazel_tools//tools/jdk:(runtime_)?toolchain_type' \\\\" \
-          -e "/bazel_build /a\  --tool_java_runtime_version=local_jdk \\\\" \
+          -e "/bazel_build /a\  --tool_java_runtime_version=local_jdk_17 \\\\" \
-          -e "/bazel_build /a\  --java_runtime_version=local_jdk \\\\" \
+          -e "/bazel_build /a\  --java_runtime_version=local_jdk_17 \\\\" \
+          -e "/bazel_build /a\  --tool_java_language_version=17 \\\\" \
+          -e "/bazel_build /a\  --java_language_version=17 \\\\" \
           -e "/bazel_build /a\  --extra_toolchains=@bazel_tools//tools/jdk:all \\\\" \
 
         # Also build parser_deploy.jar with bootstrap bazel
@@ -593,13 +588,11 @@ stdenv.mkDerivation rec {
     #
     # in the nixpkgs checkout root to exercise them locally.
     tests = callPackage ./tests.nix {
-      inherit Foundation bazel_self runJdk;
+      inherit Foundation bazel_self lockfile repoCache;
-      distDir = testRepoCache;
-      repoCache = testRepoCache;
     };
 
     updater = throw "TODO";
 
-    inherit distDir repoCache testRepoCache;
+    inherit distDir repoCache;
   };
 }

pkgs/development/tools/build-managers/bazel/bazel_7/tests.nix

@@ -1,23 +1,38 @@
 { lib
   # tooling
+, callPackage
 , fetchFromGitHub
 , newScope
 , recurseIntoAttrs
 , runCommandCC
 , stdenv
+, fetchurl
   # inputs
 , Foundation
 , bazel_self
-, distDir
 , lr
-, repoCache
-, runJdk
 , xe
 , bazel-watcher
+, lockfile
+, repoCache
 }:
 let
   inherit (stdenv.hostPlatform) isDarwin;
 
+  testsDistDir = testsRepoCache;
+  testsRepoCache = callPackage ./bazel-repository-cache.nix {
+    # We are somewhat lucky that bazel's own lockfile works for our tests.
+    # Use extraDeps if the tests need things that are not in that lockfile.
+    # But most test dependencies are bazel's builtin deps, so that at least aligns.
+    inherit lockfile;
+
+    # Take all the rules_ deps, bazel_ deps and their transitive dependencies,
+    # but none of the platform-specific binaries, as they are large and useless.
+    requiredDepNamePredicate = name:
+      null == builtins.match ".*(macos|osx|linux|win|apple|android|maven).*" name
+      && null != builtins.match "(platforms|com_google_|protobuf|rules_|bazel_).*" name ;
+  };
+
   runLocal = name: attrs: script:
     let
       attrs' = removeAttrs attrs [ "buildInputs" ];
@@ -109,35 +124,36 @@ let
     hash = "sha256-DaPKp7Sn5uvfZRjdDx6grot3g3B7trqCyL0TRIdwg98=";
   };
 
-  callBazelTest = newScope {
+  callBazelTests = args:
-    inherit runLocal bazelTest bazel-examples distDir;
+    let
-    inherit Foundation;
+      callBazelTest = newScope {
-    extraBazelArgs = ''
+        inherit runLocal bazelTest bazel-examples;
-      --repository_cache=${repoCache} \
+        inherit Foundation;
-    '';
+        bazel = bazel_self;
-    bazel = bazel_self;
+        distDir = testsDistDir;
-  };
+      };
+    in
+    recurseIntoAttrs (
+      (lib.optionalAttrs (!isDarwin) {
+        # `extracted` doesn’t work on darwin
+        shebang = callBazelTest ../shebang-test.nix (args // { inherit extracted; });
+      }) // {
+      bashTools = callBazelTest ../bash-tools-test.nix args;
+      cpp = callBazelTest ../cpp-test.nix args;
+      java = callBazelTest ../java-test.nix args;
+      pythonBinPath = callBazelTest ../python-bin-path-test.nix args;
+      protobuf = callBazelTest ../protobuf-test.nix (args // { repoCache = testsRepoCache; });
+    });
 
   bazelWithNixHacks = bazel_self.override { enableNixHacks = true; };
 
 in
-(lib.optionalAttrs (!stdenv.hostPlatform.isDarwin) {
+recurseIntoAttrs {
-  # `extracted` doesn’t work on darwin
+  distDir = testsDistDir;
-  shebang = callBazelTest ../shebang-test.nix { inherit extracted; };
+  testsRepoCache = testsRepoCache;
-}) // {
-  bashTools = callBazelTest ../bash-tools-test.nix { };
-  cpp = callBazelTest ../cpp-test.nix { };
-  java = callBazelTest ../java-test.nix { };
-  # TODO: protobuf tests just fail for now.
-  #protobuf = callBazelTest ../protobuf-test.nix { };
-  pythonBinPath = callBazelTest ../python-bin-path-test.nix { };
 
-  bashToolsWithNixHacks = callBazelTest ../bash-tools-test.nix { bazel = bazelWithNixHacks; };
+  vanilla = callBazelTests { };
-
+  withNixHacks = callBazelTests { bazel = bazelWithNixHacks; };
-  cppWithNixHacks = callBazelTest ../cpp-test.nix { bazel = bazelWithNixHacks; };
-  javaWithNixHacks = callBazelTest ../java-test.nix { bazel = bazelWithNixHacks; };
-  #protobufWithNixHacks = callBazelTest ../protobuf-test.nix { bazel = bazelWithNixHacks; };
-  pythonBinPathWithNixHacks = callBazelTest ../python-bin-path-test.nix { bazel = bazelWithNixHacks; };
 
   # downstream packages using buildBazelPackage
   # fixed-output hashes of the fetch phase need to be spot-checked manually

pkgs/development/tools/build-managers/bazel/protobuf-test.nix

@@ -13,35 +13,34 @@
 , writeScript
 , writeText
 , distDir
+, Foundation
+, callPackage
+, libtool
+, lndir
+, repoCache
+, tree
 }:
 
 let
 
-  # Use builtins.fetchurl to avoid IFD, in particular on hydra
+  lockfile = ./bazel_7/tests.MODULE.bazel.lock;
-  lockfile =
+
-    let version = "7.0.0-pre.20230917.3";
+  protocbufRepoCache = callPackage ./bazel_7/bazel-repository-cache.nix {
-    in builtins.fetchurl {
+    # We are somewhat lucky that bazel's own lockfile works for our tests.
-      url = "https://raw.githubusercontent.com/bazelbuild/bazel/${version}/MODULE.bazel.lock";
+    # Use extraDeps if the tests need things that are not in that lockfile.
-      sha256 = "0z6mlz8cn03qa40mqbw6j6kd6qyn4vgb3bb1kyidazgldxjhrz6y";
+    # But most test dependencies are bazel's builtin deps, so that at least aligns.
-    };
+    inherit lockfile;
+
+    # Take all the rules_ deps, bazel_ deps and their transitive dependencies,
+    # but none of the platform-specific binaries, as they are large and useless.
+    requiredDepNamePredicate = name:
+      null == builtins.match ".*(macos|osx|linux|win|android|maven).*" name;
+  };
 
   MODULE = writeText "MODULE.bazel" ''
-    #bazel_dep(name = "bazel_skylib", version = "1.4.1")
-    #bazel_dep(name = "protobuf", version = "21.7", repo_name = "com_google_protobuf")
-    bazel_dep(name = "protobuf", version = "21.7")
-    #bazel_dep(name = "grpc", version = "1.48.1.bcr.1", repo_name = "com_github_grpc_grpc")
-    #bazel_dep(name = "platforms", version = "0.0.7")
-    #bazel_dep(name = "rules_pkg", version = "0.9.1")
-    #bazel_dep(name = "stardoc", version = "0.5.3", repo_name = "io_bazel_skydoc")
-    #bazel_dep(name = "zstd-jni", version = "1.5.2-3.bcr.1")
-    #bazel_dep(name = "blake3", version = "1.3.3.bcr.1")
-    #bazel_dep(name = "zlib", version = "1.3")
-    #bazel_dep(name = "rules_cc", version = "0.0.8")
-    #bazel_dep(name = "rules_java", version = "6.3.1")
     bazel_dep(name = "rules_proto", version = "5.3.0-21.7")
-    #bazel_dep(name = "rules_jvm_external", version = "5.2")
+    bazel_dep(name = "protobuf", version = "21.7")
-    #bazel_dep(name = "rules_python", version = "0.24.0")
+    bazel_dep(name = "zlib", version = "1.3")
-    #bazel_dep(name = "rules_testing", version = "0.0.4")
   '';
 
   WORKSPACE = writeText "WORKSPACE" ''
@@ -106,38 +105,59 @@ let
     # See: https://github.com/bazelbuild/bazel/issues/4231
     export BAZEL_USE_CPP_ONLY_TOOLCHAIN=1
 
+    export HOMEBREW_RUBY_PATH="foo"
+
     exec "$BAZEL_REAL" "$@"
   '';
 
   workspaceDir = runLocal "our_workspace" { } (''
     mkdir $out
-    cp ${MODULE} $out/MODULE.bazel
+    cp --no-preserve=all ${MODULE} $out/MODULE.bazel
-    cp ${lockfile} $out/MODULE.bazel.lock
+    cp --no-preserve=all ${./bazel_7/tests.MODULE.bazel.lock} $out/MODULE.bazel.lock
-    cp ${WORKSPACE} $out/WORKSPACE
+    #cp ${WORKSPACE} $out/WORKSPACE
+    touch $out/WORKSPACE
     touch $out/BUILD.bazel
     mkdir $out/person
-    cp ${personProto} $out/person/person.proto
+    cp --no-preserve=all ${personProto} $out/person/person.proto
-    cp ${personBUILD} $out/person/BUILD.bazel
+    cp --no-preserve=all ${personBUILD} $out/person/BUILD.bazel
   ''
   + (lib.optionalString stdenv.isDarwin ''
+    echo 'tools bazel created'
     mkdir $out/tools
-    cp ${toolsBazel} $out/tools/bazel
+    install ${toolsBazel} $out/tools/bazel
   ''));
 
   testBazel = bazelTest {
     name = "bazel-test-protocol-buffers";
     inherit workspaceDir;
     bazelPkg = bazel;
-    buildInputs = [ (if lib.strings.versionOlder bazel.version "5.0.0" then openjdk8 else jdk11_headless) ];
+    buildInputs = [
+      (if lib.strings.versionOlder bazel.version "5.0.0" then openjdk8 else jdk11_headless)
+      tree
+      bazel
+    ]
+    ++ lib.optionals stdenv.hostPlatform.isDarwin [
+      Foundation
+      darwin.objc4
+    ];
+
     bazelScript = ''
+      # Augment bundled repository_cache with our extra paths
+      mkdir -p $PWD/.repository_cache/content_addressable
+      cp -r --no-preserve=all ${repoCache}/content_addressable/* \
+        $PWD/.repository_cache/content_addressable
+      cp -r --no-preserve=all ${protocbufRepoCache}/content_addressable/* \
+        $PWD/.repository_cache/content_addressable
+
+      tree $PWD/.repository_cache
+
       ${bazel}/bin/bazel \
         build \
+        --repository_cache=$PWD/.repository_cache \
+        --enable_bzlmod \
+        --lockfile_mode=error \
+        -s \
         --verbose_failures \
-        --curses=no \
-        --sandbox_debug \
-        --strict_java_deps=off \
-        --strict_proto_deps=off \
-        ${extraBazelArgs} \
         //... \
     '' + lib.optionalString (lib.strings.versionOlder bazel.version "5.0.0") ''
       --host_javabase='@local_jdk//:jdk' \
@@ -147,6 +167,18 @@ let
       --cxxopt=-x --cxxopt=c++ --host_cxxopt=-x --host_cxxopt=c++ \
       --linkopt=-stdlib=libc++ --host_linkopt=-stdlib=libc++ \
     '';
+      #--cxxopt=-framework --cxxopt=Foundation \
+      #--linkopt=-F${Foundation}/Library/Frameworks \
+      #--host_linkopt=-F${Foundation}/Library/Frameworks \
+
+        #--distdir=$PWD/.repository_cache \
+        #--verbose_failures \
+        #--curses=no \
+        #--sandbox_debug \
+        #--strict_java_deps=off \
+        #--strict_proto_deps=off \
+        #--repository_cache=${repoCache} \
+        #--distdir=${repoCache} \
   };
 
 in

pkgs/development/tools/build-managers/bazel/python-bin-path-test.nix

@@ -77,7 +77,7 @@ let
     bazelScript = ''
       ${bazel}/bin/bazel \
         run \
-        --repository_cache=${distDir} \
+        --distdir=${distDir} \
         ${extraBazelArgs} \
         //python:bin
     '';

pkgs/top-level/all-packages.nix

@@ -18570,7 +18570,7 @@ with pkgs;
   bazel_7 = darwin.apple_sdk_11_0.callPackage ../development/tools/build-managers/bazel/bazel_7 {
     inherit (darwin) cctools sigtool;
     inherit (darwin.apple_sdk_11_0.frameworks) CoreFoundation CoreServices Foundation IOKit;
-    buildJdk = jdk11_headless;
+    buildJdk = jdk17_headless;
     runJdk = jdk17_headless;
     stdenv = if stdenv.isDarwin then darwin.apple_sdk_11_0.stdenv
       else if stdenv.cc.isClang then llvmPackages.stdenv