cafkafk/nixpkgs
cafkafk/nixpkgs
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

llvmPackages_15.llvm: run the tests on macOS

Browse source 
there are a few parts to this:
  - adding darwin specific check deps
  - working around referencing LLVM dylibs during the checkPhase in a
    way that supports darwin
    + previously we just set `$LD_LIBRARY_PATH` and/or made some
      strategic symlinks
    + now we have LLVM's `lit` config set the appropriate env vars as
      needed (as is done for other LLVM subprojects)
    + in retrospect switching to `installCheckPhase` might have been the
      better move..
  - patching `lit` to deal with `$DYLD_LIBRARY_PATH` being purged for
    new "protected" processes

more details within.
This commit is contained in:
Rahul Butani 2023-01-08 03:47:47 -08:00
parent 6d0c87602f
commit c7231c0b6d
No known key found for this signature in database
4 changed files with 180 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

69
pkgs/development/compilers/llvm/15/llvm/default.nix
View file

@ -15,9 +15,11 @@
, release_version
, zlib
, which
, sysctl
, buildLlvmTools
, debugVersion ? false
, doCheck ? true
, doCheck ? (!stdenv.isx86_32 /* TODO: why */) && (!stdenv.hostPlatform.isMusl)
&& (stdenv.hostPlatform == stdenv.buildPlatform)
, enableManpages ? false
, enableSharedLibraries ? !stdenv.hostPlatform.isStatic
, enablePFM ? stdenv.isLinux /* PFM only supports Linux */
@ -89,7 +91,53 @@ in stdenv.mkDerivation (rec {
patches = [
./gnu-install-dirs.patch
] ++ lib.optional enablePolly ./gnu-install-dirs-polly.patch;
# Running the tests involves invoking binaries (like `opt`) that depend on
# the LLVM dylibs and reference them by absolute install path (i.e. their
# nix store path).
#
# Because we have not yet run the install phase (we're running these tests
# as part of `checkPhase` instead of `installCheckPhase`) these absolute
# paths do not exist yet; to work around this we point the loader (`ld` on
# unix, `dyld` on macOS) at the `lib` directory which will later become this
# package's `lib` output.
#
# Previously we would just set `LD_LIBRARY_PATH` to include the build `lib`
# dir but:
# - this doesn't generalize well to other platforms; `lit` doesn't forward
# `DYLD_LIBRARY_PATH` (macOS):
# + https://github.com/llvm/llvm-project/blob/0d89963df354ee309c15f67dc47c8ab3cb5d0fb2/llvm/utils/lit/lit/TestingConfig.py#L26
# - even if `lit` forwarded this env var, we actually cannot set
# `DYLD_LIBRARY_PATH` in the child processes `lit` launches because
# `DYLD_LIBRARY_PATH` (and `DYLD_FALLBACK_LIBRARY_PATH`) is cleared for
# "protected processes" (i.e. the python interpreter that runs `lit`):
# https://stackoverflow.com/a/35570229
# - other LLVM subprojects deal with this issue by having their `lit`
# configuration set these env vars for us; it makes sense to do the same
# for LLVM:
# + https://github.com/llvm/llvm-project/blob/4c106cfdf7cf7eec861ad3983a3dd9a9e8f3a8ae/clang-tools-extra/test/Unit/lit.cfg.py#L22-L31
#
# !!! TODO: look into upstreaming this patch
./llvm-lit-cfg-add-libs-to-dylib-path.patch
# `lit` has a mode where it executes run lines as a shell script which is
# constructs; this is problematic for macOS because it means that there's
# another process in between `lit` and the binaries being tested. As noted
# above, this means that `DYLD_LIBRARY_PATH` is cleared which means that our
# tests fail with dyld errors.
#
# To get around this we patch `lit` to reintroduce `DYLD_LIBRARY_PATH`, when
# present in the test configuration.
#
# It's not clear to me why this isn't an issue for LLVM developers running
# on macOS (nothing about this _seems_ nix specific)..
./lit-shell-script-runner-set-dyld-library-path.patch
] ++ lib.optionals enablePolly [
./gnu-install-dirs-polly.patch
# Just like the `llvm-lit-cfg` patch, but for `polly`.
./polly-lit-cfg-add-libs-to-dylib-path.patch
];
postPatch = optionalString stdenv.isDarwin ''
substituteInPlace cmake/modules/AddLLVM.cmake \
@ -134,12 +182,6 @@ in stdenv.mkDerivation (rec {
)
'';
# hacky fix: created binaries need to be run before installation
preBuild = ''
mkdir -p $out/
ln -sv $PWD/lib $out
'';
# E.g. mesa.drivers use the build-id as a cache key (see #93946):
LDFLAGS = optionalString (enableSharedLibraries && !stdenv.isDarwin) "-Wl,--build-id=sha1";
@ -217,14 +259,6 @@ in stdenv.mkDerivation (rec {
)
];
postBuild = ''
rm -fR $out
'';
preCheck = ''
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH''${LD_LIBRARY_PATH:+:}$PWD/lib
'';
postInstall = ''
mkdir -p $python/share
mv $out/share/opt-viewer $python/share/opt-viewer
@ -243,8 +277,7 @@ in stdenv.mkDerivation (rec {
cp NATIVE/bin/llvm-config $dev/bin/llvm-config-native
'';
doCheck = stdenv.isLinux && (!stdenv.isx86_32) && (!stdenv.hostPlatform.isMusl)
&& (stdenv.hostPlatform == stdenv.buildPlatform);
inherit doCheck;
checkTarget = "check-all";

26
pkgs/development/compilers/llvm/15/llvm/lit-shell-script-runner-set-dyld-library-path.patch Normal file
View file

@ -0,0 +1,26 @@
diff --git a/utils/lit/lit/TestRunner.py b/utils/lit/lit/TestRunner.py
index 0242e0b75af3..d732011306f7 100644
--- a/utils/lit/lit/TestRunner.py
+++ b/utils/lit/lit/TestRunner.py
@@ -1029,6 +1029,12 @@ def executeScript(test, litConfig, tmpBase, commands, cwd):
f.write('@echo off\n')
f.write('\n@if %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands))
else:
+ # This env var is *purged* when invoking subprocesses so we have to
+ # manually set it from within the bash script in order for the commands
+ # in run lines to see this var:
+ if "DYLD_LIBRARY_PATH" in test.config.environment:
+ f.write(f'export DYLD_LIBRARY_PATH="{test.config.environment["DYLD_LIBRARY_PATH"]}"\n')
+
for i, ln in enumerate(commands):
match = re.match(kPdbgRegex, ln)
if match:
@@ -1363,7 +1369,7 @@ def applySubstitutions(script, substitutions, conditions={},
return processed
process = processLine if recursion_limit is None else processLineToFixedPoint
-
+
return [unescapePercents(process(ln)) for ln in script]

79
pkgs/development/compilers/llvm/15/llvm/llvm-lit-cfg-add-libs-to-dylib-path.patch Normal file
View file

@ -0,0 +1,79 @@
diff --git a/test/Unit/lit.cfg.py b/test/Unit/lit.cfg.py
index 81e8dc04acea..479ff95681e2 100644
--- a/test/Unit/lit.cfg.py
+++ b/test/Unit/lit.cfg.py
@@ -3,6 +3,7 @@
# Configuration file for the 'lit' test runner.
import os
+import platform
import subprocess
import lit.formats
@@ -55,3 +56,26 @@ if sys.platform in ['win32', 'cygwin'] and os.path.isdir(config.shlibdir):
# Win32 may use %SYSTEMDRIVE% during file system shell operations, so propogate.
if sys.platform == 'win32' and 'SYSTEMDRIVE' in os.environ:
config.environment['SYSTEMDRIVE'] = os.environ['SYSTEMDRIVE']
+
+# Add the LLVM dynamic libs to the platform-specific loader search path env var:
+#
+# TODO: this is copied from `clang`'s `lit.cfg.py`; should unify..
+def find_shlibpath_var():
+ if platform.system() in ['Linux', 'FreeBSD', 'NetBSD', 'OpenBSD', 'SunOS']:
+ yield 'LD_LIBRARY_PATH'
+ elif platform.system() == 'Darwin':
+ yield 'DYLD_LIBRARY_PATH'
+ elif platform.system() == 'Windows':
+ yield 'PATH'
+ elif platform.system() == 'AIX':
+ yield 'LIBPATH'
+
+for shlibpath_var in find_shlibpath_var():
+ shlibpath = os.path.pathsep.join(
+ (config.shlibdir,
+ config.environment.get(shlibpath_var, '')))
+ config.environment[shlibpath_var] = shlibpath
+ break
+else:
+ lit_config.warning("unable to inject shared library path on '{}'"
+ .format(platform.system()))
diff --git a/test/lit.cfg.py b/test/lit.cfg.py
index 75a38b4c5dad..856fc75c9d74 100644
--- a/test/lit.cfg.py
+++ b/test/lit.cfg.py
@@ -42,6 +42,26 @@ llvm_config.with_environment('PATH', config.llvm_tools_dir, append_path=True)
llvm_config.with_system_environment(
['HOME', 'INCLUDE', 'LIB', 'TMP', 'TEMP'])
+# Add the LLVM dynamic libs to the platform-specific loader search path env var:
+#
+# TODO: this is copied from `clang`'s `lit.cfg.py`; should unify..
+def find_shlibpath_var():
+ if platform.system() in ['Linux', 'FreeBSD', 'NetBSD', 'OpenBSD', 'SunOS']:
+ yield 'LD_LIBRARY_PATH'
+ elif platform.system() == 'Darwin':
+ yield 'DYLD_LIBRARY_PATH'
+ elif platform.system() == 'Windows':
+ yield 'PATH'
+ elif platform.system() == 'AIX':
+ yield 'LIBPATH'
+
+for shlibpath_var in find_shlibpath_var():
+ shlibpath = config.llvm_shlib_dir
+ llvm_config.with_environment(shlibpath_var, shlibpath, append_path = True)
+ break
+else:
+ lit_config.warning("unable to inject shared library path on '{}'"
+ .format(platform.system()))
# Set up OCAMLPATH to include newly built OCaml libraries.
top_ocaml_lib = os.path.join(config.llvm_lib_dir, 'ocaml')
@@ -318,7 +338,7 @@ def have_cxx_shared_library():
try:
readobj_cmd = subprocess.Popen(
- [readobj_exe, '--needed-libs', readobj_exe], stdout=subprocess.PIPE)
+ [readobj_exe, '--needed-libs', readobj_exe], stdout=subprocess.PIPE, env=config.environment)
except OSError:
print('could not exec llvm-readobj')
return False

24
pkgs/development/compilers/llvm/15/llvm/polly-lit-cfg-add-libs-to-dylib-path.patch Normal file
View file

@ -0,0 +1,24 @@
diff --git a/tools/polly/test/lit.cfg b/tools/polly/test/lit.cfg
index 41e3a589c61e..09f3b17498b0 100644
--- a/tools/polly/test/lit.cfg
+++ b/tools/polly/test/lit.cfg
@@ -36,9 +36,17 @@ base_paths = [config.llvm_tools_dir, config.environment['PATH']]
path = os.path.pathsep.join(base_paths + config.extra_paths)
config.environment['PATH'] = path
+# (Copied from polly/test/Unit/lit.cfg)
+if platform.system() == 'Darwin':
+ shlibpath_var = 'DYLD_LIBRARY_PATH'
+elif platform.system() == 'Windows':
+ shlibpath_var = 'PATH'
+else:
+ shlibpath_var = 'LD_LIBRARY_PATH'
+
path = os.path.pathsep.join((config.llvm_libs_dir,
- config.environment.get('LD_LIBRARY_PATH','')))
-config.environment['LD_LIBRARY_PATH'] = path
+ config.environment.get(shlibpath_var,'')))
+config.environment[shlibpath_var] = path
llvm_config.use_default_substitutions()