Rework repository_cache, lockfile generation, and protobuf tests as a whole
This commit is contained in:
Guillaume Maudoux
parent
229bce1095
commit
be69c186cc
7 changed files with 2949 additions and 86 deletions
|
|
@ -8,9 +8,11 @@
|
|||
, requiredDepNamePredicate ? _: true
|
||||
# Extra deps in the form of derivations producing a "single file" output path
|
||||
, extraDeps ? [ ]
|
||||
, canonicalIds ? true
|
||||
}:
|
||||
let
|
||||
modules = builtins.fromJSON (builtins.readFile lockfile);
|
||||
modulesVersion = modules.lockFileVersion;
|
||||
|
||||
# a foldl' for json values
|
||||
foldlJSON = op: acc: value:
|
||||
|
|
@ -31,7 +33,10 @@ let
|
|||
builtins.seq acc' children;
|
||||
|
||||
# remove the "--" prefix, abusing undocumented negative substring length
|
||||
sanitize = builtins.substring 2 (-1);
|
||||
sanitize = str:
|
||||
if modulesVersion < 3
|
||||
then builtins.substring 2 (-1) str
|
||||
else str;
|
||||
|
||||
extract_source = f: acc: value:
|
||||
# We take any "attributes" object that has a "sha256" field. Every value
|
||||
|
|
@ -59,6 +64,7 @@ let
|
|||
sha256 = hash;
|
||||
passthru.sha256 = hash;
|
||||
passthru.source_name = name;
|
||||
passthru.urls = urls;
|
||||
};
|
||||
};
|
||||
insert = acc: hash: urls: name:
|
||||
|
|
@ -76,7 +82,8 @@ let
|
|||
if builtins.isAttrs value && value ? attributes
|
||||
&& (attrs ? sha256 || attrs ? integrity)
|
||||
&& f attrs.name
|
||||
then accWithNewSource else acc;
|
||||
then accWithNewSource
|
||||
else acc;
|
||||
|
||||
requiredSourcePredicate = n: requiredDepNamePredicate (sanitize n);
|
||||
requiredDeps = foldlJSON (extract_source requiredSourcePredicate) { } modules;
|
||||
|
|
@ -89,10 +96,23 @@ let
|
|||
(drv: ''
|
||||
filename=$(basename "${lib.head drv.urls}")
|
||||
echo Bundling $filename ${lib.optionalString (drv?source_name) "from ${drv.source_name}"}
|
||||
|
||||
# 1. --repository_cache format:
|
||||
# 1.a. A file unde a content-hash directory
|
||||
hash=$(${rnix-hashes}/bin/rnix-hashes --encoding BASE16 ${drv.sha256} | cut -f 2)
|
||||
mkdir -p content_addressable/sha256/$hash
|
||||
ln -sfn ${drv} content_addressable/sha256/$hash/file
|
||||
# Expect file name conflicts
|
||||
|
||||
# 1.b. a canonicalId marker based on the download urls
|
||||
# Bazel uses these to avoid reusing a stale hash when the urls have changed.
|
||||
canonicalId="${lib.concatStringsSep " " drv.urls}"
|
||||
canonicalIdHash=$(echo -n "$canonicalId" | sha256sum | cut -d" " -f1)
|
||||
echo -n "$canonicalId" > content_addressable/sha256/$hash/id-$canonicalIdHash
|
||||
|
||||
# 2. --distdir format:
|
||||
# Just a file with the right basename
|
||||
# Mostly to keep old tests happy, and because symlinks cost nothing.
|
||||
# This is brittle because of expected file name conflicts
|
||||
ln -sn ${drv} $filename || true
|
||||
'')
|
||||
(builtins.attrValues requiredDeps ++ extraDeps)
|
||||
|
|
|
|||
|
|
@ -58,39 +58,32 @@
|
|||
}@args:
|
||||
|
||||
let
|
||||
version = "7.0.0-pre.20230917.3";
|
||||
version = "7.0.0-pre.20231011.2";
|
||||
sourceRoot = ".";
|
||||
|
||||
src = fetchurl {
|
||||
url = "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-dist.zip";
|
||||
hash = "sha256-sxITaivcJRrQrL+zZtdZohesNgmDtQysIG3BS8SFZd4=";
|
||||
hash = "sha256-3kwNnFPGyRLBKSSzsa+pTNAHowH+fLjtwxiSY8RkbQc=";
|
||||
};
|
||||
|
||||
# Use builtins.fetchurl to avoid IFD, in particular on hydra
|
||||
lockfile = builtins.fetchurl {
|
||||
url = "https://raw.githubusercontent.com/bazelbuild/bazel/${version}/MODULE.bazel.lock";
|
||||
sha256 = "0z6mlz8cn03qa40mqbw6j6kd6qyn4vgb3bb1kyidazgldxjhrz6y";
|
||||
sha256 = "sha256:0wdwkfq3mrg4dxvmq2prdsj1i5jyql8jc2wf3wx948cnc10pbfy7";
|
||||
};
|
||||
|
||||
# Two-in-one format
|
||||
distDir = repoCache;
|
||||
repoCache = callPackage ./bazel-repository-cache.nix {
|
||||
inherit lockfile;
|
||||
# We use the release tarball that already has everything bundled,
|
||||
# But we need one extra dep required by our nonprebuilt java toolchains.
|
||||
|
||||
# We use the release tarball that already has everything bundled so we
|
||||
# should not need any extra external deps. But our nonprebuilt java
|
||||
# toolchains hack needs just one non bundled dep.
|
||||
requiredDepNamePredicate = name:
|
||||
null != builtins.match "rules_java~.*~toolchains~remote_java_tools" name;
|
||||
};
|
||||
|
||||
# Two-in-one format
|
||||
testRepoCache = callPackage ./bazel-repository-cache.nix {
|
||||
inherit lockfile;
|
||||
# We use the release tarball that already has everything bundled,
|
||||
# But we need one extra dep required by our nonprebuilt java toolchains.
|
||||
requiredDepNamePredicate = name:
|
||||
null == builtins.match ".*(macos|osx|linux|win|apple|android).*" name;
|
||||
};
|
||||
|
||||
defaultShellUtils =
|
||||
# Keep this list conservative. For more exotic tools, prefer to use
|
||||
# @rules_nixpkgs to pull in tools from the nix repository. Example:
|
||||
|
|
@ -379,8 +372,10 @@ stdenv.mkDerivation rec {
|
|||
-e "/bazel_build /a\ --experimental_strict_java_deps=off \\\\" \
|
||||
-e "/bazel_build /a\ --strict_proto_deps=off \\\\" \
|
||||
-e "/bazel_build /a\ --toolchain_resolution_debug='@bazel_tools//tools/jdk:(runtime_)?toolchain_type' \\\\" \
|
||||
-e "/bazel_build /a\ --tool_java_runtime_version=local_jdk \\\\" \
|
||||
-e "/bazel_build /a\ --java_runtime_version=local_jdk \\\\" \
|
||||
-e "/bazel_build /a\ --tool_java_runtime_version=local_jdk_17 \\\\" \
|
||||
-e "/bazel_build /a\ --java_runtime_version=local_jdk_17 \\\\" \
|
||||
-e "/bazel_build /a\ --tool_java_language_version=17 \\\\" \
|
||||
-e "/bazel_build /a\ --java_language_version=17 \\\\" \
|
||||
-e "/bazel_build /a\ --extra_toolchains=@bazel_tools//tools/jdk:all \\\\" \
|
||||
|
||||
# Also build parser_deploy.jar with bootstrap bazel
|
||||
|
|
@ -593,13 +588,11 @@ stdenv.mkDerivation rec {
|
|||
#
|
||||
# in the nixpkgs checkout root to exercise them locally.
|
||||
tests = callPackage ./tests.nix {
|
||||
inherit Foundation bazel_self runJdk;
|
||||
distDir = testRepoCache;
|
||||
repoCache = testRepoCache;
|
||||
inherit Foundation bazel_self lockfile repoCache;
|
||||
};
|
||||
|
||||
updater = throw "TODO";
|
||||
|
||||
inherit distDir repoCache testRepoCache;
|
||||
inherit distDir repoCache;
|
||||
};
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,23 +1,38 @@
|
|||
{ lib
|
||||
# tooling
|
||||
, callPackage
|
||||
, fetchFromGitHub
|
||||
, newScope
|
||||
, recurseIntoAttrs
|
||||
, runCommandCC
|
||||
, stdenv
|
||||
, fetchurl
|
||||
# inputs
|
||||
, Foundation
|
||||
, bazel_self
|
||||
, distDir
|
||||
, lr
|
||||
, repoCache
|
||||
, runJdk
|
||||
, xe
|
||||
, bazel-watcher
|
||||
, lockfile
|
||||
, repoCache
|
||||
}:
|
||||
let
|
||||
inherit (stdenv.hostPlatform) isDarwin;
|
||||
|
||||
testsDistDir = testsRepoCache;
|
||||
testsRepoCache = callPackage ./bazel-repository-cache.nix {
|
||||
# We are somewhat lucky that bazel's own lockfile works for our tests.
|
||||
# Use extraDeps if the tests need things that are not in that lockfile.
|
||||
# But most test dependencies are bazel's builtin deps, so that at least aligns.
|
||||
inherit lockfile;
|
||||
|
||||
# Take all the rules_ deps, bazel_ deps and their transitive dependencies,
|
||||
# but none of the platform-specific binaries, as they are large and useless.
|
||||
requiredDepNamePredicate = name:
|
||||
null == builtins.match ".*(macos|osx|linux|win|apple|android|maven).*" name
|
||||
&& null != builtins.match "(platforms|com_google_|protobuf|rules_|bazel_).*" name ;
|
||||
};
|
||||
|
||||
runLocal = name: attrs: script:
|
||||
let
|
||||
attrs' = removeAttrs attrs [ "buildInputs" ];
|
||||
|
|
@ -109,35 +124,36 @@ let
|
|||
hash = "sha256-DaPKp7Sn5uvfZRjdDx6grot3g3B7trqCyL0TRIdwg98=";
|
||||
};
|
||||
|
||||
callBazelTest = newScope {
|
||||
inherit runLocal bazelTest bazel-examples distDir;
|
||||
inherit Foundation;
|
||||
extraBazelArgs = ''
|
||||
--repository_cache=${repoCache} \
|
||||
'';
|
||||
bazel = bazel_self;
|
||||
};
|
||||
callBazelTests = args:
|
||||
let
|
||||
callBazelTest = newScope {
|
||||
inherit runLocal bazelTest bazel-examples;
|
||||
inherit Foundation;
|
||||
bazel = bazel_self;
|
||||
distDir = testsDistDir;
|
||||
};
|
||||
in
|
||||
recurseIntoAttrs (
|
||||
(lib.optionalAttrs (!isDarwin) {
|
||||
# `extracted` doesn’t work on darwin
|
||||
shebang = callBazelTest ../shebang-test.nix (args // { inherit extracted; });
|
||||
}) // {
|
||||
bashTools = callBazelTest ../bash-tools-test.nix args;
|
||||
cpp = callBazelTest ../cpp-test.nix args;
|
||||
java = callBazelTest ../java-test.nix args;
|
||||
pythonBinPath = callBazelTest ../python-bin-path-test.nix args;
|
||||
protobuf = callBazelTest ../protobuf-test.nix (args // { repoCache = testsRepoCache; });
|
||||
});
|
||||
|
||||
bazelWithNixHacks = bazel_self.override { enableNixHacks = true; };
|
||||
|
||||
in
|
||||
(lib.optionalAttrs (!stdenv.hostPlatform.isDarwin) {
|
||||
# `extracted` doesn’t work on darwin
|
||||
shebang = callBazelTest ../shebang-test.nix { inherit extracted; };
|
||||
}) // {
|
||||
bashTools = callBazelTest ../bash-tools-test.nix { };
|
||||
cpp = callBazelTest ../cpp-test.nix { };
|
||||
java = callBazelTest ../java-test.nix { };
|
||||
# TODO: protobuf tests just fail for now.
|
||||
#protobuf = callBazelTest ../protobuf-test.nix { };
|
||||
pythonBinPath = callBazelTest ../python-bin-path-test.nix { };
|
||||
recurseIntoAttrs {
|
||||
distDir = testsDistDir;
|
||||
testsRepoCache = testsRepoCache;
|
||||
|
||||
bashToolsWithNixHacks = callBazelTest ../bash-tools-test.nix { bazel = bazelWithNixHacks; };
|
||||
|
||||
cppWithNixHacks = callBazelTest ../cpp-test.nix { bazel = bazelWithNixHacks; };
|
||||
javaWithNixHacks = callBazelTest ../java-test.nix { bazel = bazelWithNixHacks; };
|
||||
#protobufWithNixHacks = callBazelTest ../protobuf-test.nix { bazel = bazelWithNixHacks; };
|
||||
pythonBinPathWithNixHacks = callBazelTest ../python-bin-path-test.nix { bazel = bazelWithNixHacks; };
|
||||
vanilla = callBazelTests { };
|
||||
withNixHacks = callBazelTests { bazel = bazelWithNixHacks; };
|
||||
|
||||
# downstream packages using buildBazelPackage
|
||||
# fixed-output hashes of the fetch phase need to be spot-checked manually
|
||||
|
|
|
|||
|
|
@ -13,35 +13,34 @@
|
|||
, writeScript
|
||||
, writeText
|
||||
, distDir
|
||||
, Foundation
|
||||
, callPackage
|
||||
, libtool
|
||||
, lndir
|
||||
, repoCache
|
||||
, tree
|
||||
}:
|
||||
|
||||
let
|
||||
|
||||
# Use builtins.fetchurl to avoid IFD, in particular on hydra
|
||||
lockfile =
|
||||
let version = "7.0.0-pre.20230917.3";
|
||||
in builtins.fetchurl {
|
||||
url = "https://raw.githubusercontent.com/bazelbuild/bazel/${version}/MODULE.bazel.lock";
|
||||
sha256 = "0z6mlz8cn03qa40mqbw6j6kd6qyn4vgb3bb1kyidazgldxjhrz6y";
|
||||
};
|
||||
lockfile = ./bazel_7/tests.MODULE.bazel.lock;
|
||||
|
||||
protocbufRepoCache = callPackage ./bazel_7/bazel-repository-cache.nix {
|
||||
# We are somewhat lucky that bazel's own lockfile works for our tests.
|
||||
# Use extraDeps if the tests need things that are not in that lockfile.
|
||||
# But most test dependencies are bazel's builtin deps, so that at least aligns.
|
||||
inherit lockfile;
|
||||
|
||||
# Take all the rules_ deps, bazel_ deps and their transitive dependencies,
|
||||
# but none of the platform-specific binaries, as they are large and useless.
|
||||
requiredDepNamePredicate = name:
|
||||
null == builtins.match ".*(macos|osx|linux|win|android|maven).*" name;
|
||||
};
|
||||
|
||||
MODULE = writeText "MODULE.bazel" ''
|
||||
#bazel_dep(name = "bazel_skylib", version = "1.4.1")
|
||||
#bazel_dep(name = "protobuf", version = "21.7", repo_name = "com_google_protobuf")
|
||||
bazel_dep(name = "protobuf", version = "21.7")
|
||||
#bazel_dep(name = "grpc", version = "1.48.1.bcr.1", repo_name = "com_github_grpc_grpc")
|
||||
#bazel_dep(name = "platforms", version = "0.0.7")
|
||||
#bazel_dep(name = "rules_pkg", version = "0.9.1")
|
||||
#bazel_dep(name = "stardoc", version = "0.5.3", repo_name = "io_bazel_skydoc")
|
||||
#bazel_dep(name = "zstd-jni", version = "1.5.2-3.bcr.1")
|
||||
#bazel_dep(name = "blake3", version = "1.3.3.bcr.1")
|
||||
#bazel_dep(name = "zlib", version = "1.3")
|
||||
#bazel_dep(name = "rules_cc", version = "0.0.8")
|
||||
#bazel_dep(name = "rules_java", version = "6.3.1")
|
||||
bazel_dep(name = "rules_proto", version = "5.3.0-21.7")
|
||||
#bazel_dep(name = "rules_jvm_external", version = "5.2")
|
||||
#bazel_dep(name = "rules_python", version = "0.24.0")
|
||||
#bazel_dep(name = "rules_testing", version = "0.0.4")
|
||||
bazel_dep(name = "protobuf", version = "21.7")
|
||||
bazel_dep(name = "zlib", version = "1.3")
|
||||
'';
|
||||
|
||||
WORKSPACE = writeText "WORKSPACE" ''
|
||||
|
|
@ -106,38 +105,59 @@ let
|
|||
# See: https://github.com/bazelbuild/bazel/issues/4231
|
||||
export BAZEL_USE_CPP_ONLY_TOOLCHAIN=1
|
||||
|
||||
export HOMEBREW_RUBY_PATH="foo"
|
||||
|
||||
exec "$BAZEL_REAL" "$@"
|
||||
'';
|
||||
|
||||
workspaceDir = runLocal "our_workspace" { } (''
|
||||
mkdir $out
|
||||
cp ${MODULE} $out/MODULE.bazel
|
||||
cp ${lockfile} $out/MODULE.bazel.lock
|
||||
cp ${WORKSPACE} $out/WORKSPACE
|
||||
cp --no-preserve=all ${MODULE} $out/MODULE.bazel
|
||||
cp --no-preserve=all ${./bazel_7/tests.MODULE.bazel.lock} $out/MODULE.bazel.lock
|
||||
#cp ${WORKSPACE} $out/WORKSPACE
|
||||
touch $out/WORKSPACE
|
||||
touch $out/BUILD.bazel
|
||||
mkdir $out/person
|
||||
cp ${personProto} $out/person/person.proto
|
||||
cp ${personBUILD} $out/person/BUILD.bazel
|
||||
cp --no-preserve=all ${personProto} $out/person/person.proto
|
||||
cp --no-preserve=all ${personBUILD} $out/person/BUILD.bazel
|
||||
''
|
||||
+ (lib.optionalString stdenv.isDarwin ''
|
||||
echo 'tools bazel created'
|
||||
mkdir $out/tools
|
||||
cp ${toolsBazel} $out/tools/bazel
|
||||
install ${toolsBazel} $out/tools/bazel
|
||||
''));
|
||||
|
||||
testBazel = bazelTest {
|
||||
name = "bazel-test-protocol-buffers";
|
||||
inherit workspaceDir;
|
||||
bazelPkg = bazel;
|
||||
buildInputs = [ (if lib.strings.versionOlder bazel.version "5.0.0" then openjdk8 else jdk11_headless) ];
|
||||
buildInputs = [
|
||||
(if lib.strings.versionOlder bazel.version "5.0.0" then openjdk8 else jdk11_headless)
|
||||
tree
|
||||
bazel
|
||||
]
|
||||
++ lib.optionals stdenv.hostPlatform.isDarwin [
|
||||
Foundation
|
||||
darwin.objc4
|
||||
];
|
||||
|
||||
bazelScript = ''
|
||||
# Augment bundled repository_cache with our extra paths
|
||||
mkdir -p $PWD/.repository_cache/content_addressable
|
||||
cp -r --no-preserve=all ${repoCache}/content_addressable/* \
|
||||
$PWD/.repository_cache/content_addressable
|
||||
cp -r --no-preserve=all ${protocbufRepoCache}/content_addressable/* \
|
||||
$PWD/.repository_cache/content_addressable
|
||||
|
||||
tree $PWD/.repository_cache
|
||||
|
||||
${bazel}/bin/bazel \
|
||||
build \
|
||||
--repository_cache=$PWD/.repository_cache \
|
||||
--enable_bzlmod \
|
||||
--lockfile_mode=error \
|
||||
-s \
|
||||
--verbose_failures \
|
||||
--curses=no \
|
||||
--sandbox_debug \
|
||||
--strict_java_deps=off \
|
||||
--strict_proto_deps=off \
|
||||
${extraBazelArgs} \
|
||||
//... \
|
||||
'' + lib.optionalString (lib.strings.versionOlder bazel.version "5.0.0") ''
|
||||
--host_javabase='@local_jdk//:jdk' \
|
||||
|
|
@ -147,6 +167,18 @@ let
|
|||
--cxxopt=-x --cxxopt=c++ --host_cxxopt=-x --host_cxxopt=c++ \
|
||||
--linkopt=-stdlib=libc++ --host_linkopt=-stdlib=libc++ \
|
||||
'';
|
||||
#--cxxopt=-framework --cxxopt=Foundation \
|
||||
#--linkopt=-F${Foundation}/Library/Frameworks \
|
||||
#--host_linkopt=-F${Foundation}/Library/Frameworks \
|
||||
|
||||
#--distdir=$PWD/.repository_cache \
|
||||
#--verbose_failures \
|
||||
#--curses=no \
|
||||
#--sandbox_debug \
|
||||
#--strict_java_deps=off \
|
||||
#--strict_proto_deps=off \
|
||||
#--repository_cache=${repoCache} \
|
||||
#--distdir=${repoCache} \
|
||||
};
|
||||
|
||||
in
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ let
|
|||
bazelScript = ''
|
||||
${bazel}/bin/bazel \
|
||||
run \
|
||||
--repository_cache=${distDir} \
|
||||
--distdir=${distDir} \
|
||||
${extraBazelArgs} \
|
||||
//python:bin
|
||||
'';
|
||||
|
|
|
|||
|
|
@ -18570,7 +18570,7 @@ with pkgs;
|
|||
bazel_7 = darwin.apple_sdk_11_0.callPackage ../development/tools/build-managers/bazel/bazel_7 {
|
||||
inherit (darwin) cctools sigtool;
|
||||
inherit (darwin.apple_sdk_11_0.frameworks) CoreFoundation CoreServices Foundation IOKit;
|
||||
buildJdk = jdk11_headless;
|
||||
buildJdk = jdk17_headless;
|
||||
runJdk = jdk17_headless;
|
||||
stdenv = if stdenv.isDarwin then darwin.apple_sdk_11_0.stdenv
|
||||
else if stdenv.cc.isClang then llvmPackages.stdenv
|
||||
|
|
|
|||
Loading…
Reference in a new issue