texlive.buildTeXLivePackage: switch to fake multi-output derivations for TeX Live packages

This commit is contained in:
Vincenzo Mantova 2023-09-03 12:02:33 +01:00
parent a4ea714c80
commit 333a351b28
6 changed files with 214 additions and 132 deletions

View file

@ -648,10 +648,13 @@ rec {
# this is effectively an eval-time assertion, converted into a derivation for
# ease of testing
fixedHashes = with lib; let
combine = findFirst (p: (head p.pkgs).pname == "combine") { pkgs = []; } (head texlive.collection-latexextra.pkgs).tlDeps;
all = concatLists (map (p: p.pkgs or []) (attrValues (removeAttrs texlive [ "bin" "combine" "combined" "tlpdb" ]))) ++ combine.pkgs;
fods = filter (p: isDerivation p && p.tlType != "bin") all;
errorText = concatMapStrings (p: optionalString (! p ? outputHash) "${p.pname + optionalString (p.tlType != "run") ("." + p.tlType)} does not have a fixed output hash\n") fods;
fods = lib.concatMap
(p: lib.optional (p ? tex) p.tex
++ lib.optional (p ? texdoc) p.texdoc
++ lib.optional (p ? texsource) p.texsource
++ lib.optional (p ? tlpkg) p.tlpkg)
(attrValues texlive.pkgs);
errorText = concatMapStrings (p: optionalString (! p ? outputHash) "${p.pname}-${p.tlOutputName} does not have a fixed output hash\n") fods;
in runCommand "texlive-test-fixed-hashes" {
inherit errorText;
passAsFile = [ "errorText" ];

View file

@ -15,17 +15,27 @@
, texliveBinaries
}:
/* Convert an attribute set extracted from tlpdb.nix (with the deps attribute
already processed) to a fake multi-output derivation with possible outputs
[ "tex" "texdoc" "texsource" "tlpkg" "out" "man" "info" ]
*/
# TODO stabilise a generic interface decoupled from the finer details of the
# translation from texlive.tlpdb to tlpdb.nix
{ pname
, revision
, version ? toString revision
, extraRevision ? ""
, extraVersion ? ""
, sha512
, mirrors
, extraVersion ? ""
, fixedHashes ? { }
, postUnpack ? ""
, postFixup ? ""
, stripPrefix ? 1
, license ? [ ]
, hasHyphens ? false
, hasInfo ? false
, hasManpages ? false
, hasRunfiles ? false
, hasTlpkg ? false
@ -34,112 +44,165 @@
}@args:
let
meta = { license = map (x: lib.licenses.${x}) license; };
commonPassthru = {
inherit pname revision version;
} // lib.optionalAttrs (args ? extraRevision) {
inherit (args) extraRevision;
# common metadata
name = "${pname}-${version}${extraVersion}";
meta = {
license = map (x: lib.licenses.${x}) license;
# TeX Live packages should not be installed directly into the user profile
outputsToInstall = [ ];
};
hasBinfiles = args ? binfiles && args.binfiles != [ ];
hasDocfiles = sha512 ? doc;
hasSource = sha512 ? source;
# emulate drv.all, drv.outputs lists
all = lib.optional hasBinfiles bin ++
lib.optional hasRunfiles tex ++
lib.optional hasDocfiles texdoc ++
lib.optional hasSource texsource ++
lib.optional hasTlpkg tlpkg ++
lib.optional hasManpages man ++
lib.optional hasInfo info;
outputs = lib.catAttrs "tlOutputName" all;
mainDrv = if hasBinfiles then bin
else if hasRunfiles then tex
else if hasTlpkg then tlpkg
else if hasDocfiles then texdoc
else if hasSource then texsource
else tex; # fall back to attrset tex if there is no derivation
# emulate multi-output derivation plus additional metadata
# (out is handled in mkContainer)
passthru = {
inherit all outputs pname;
revision = toString revision + extraRevision;
version = version + extraVersion;
outputSpecified = true;
inherit tex;
} // lib.optionalAttrs (args ? deps) { tlDeps = args.deps; }
// lib.optionalAttrs (args ? formats) { inherit (args) formats; }
// lib.optionalAttrs hasHyphens { inherit hasHyphens; }
// lib.optionalAttrs (args ? postactionScript) { inherit (args) postactionScript; }
// lib.optionalAttrs hasDocfiles { texdoc = texdoc; }
// lib.optionalAttrs hasSource { texsource = texsource; }
// lib.optionalAttrs hasTlpkg { tlpkg = tlpkg; }
// lib.optionalAttrs hasManpages { man = man; }
// lib.optionalAttrs hasInfo { info = info; };
# build run, doc, source, tlpkg containers
mkContainer = tlType: passthru: sha512:
mkContainer = tlType: tlOutputName: sha512:
let
# NOTE: the fixed naming scheme must match generated-fixed-hashes.nix
# the basename used by upstream (without ".tar.xz" suffix)
urlName = pname + (lib.optionalString (tlType != "run" && tlType != "tlpkg") ".${tlType}");
# name + version for the derivation
tlName = urlName + (lib.optionalString (tlType == "tlpkg") ".tlpkg") + "-${version}${extraVersion}";
fixedHash = fixedHashes.${tlType} or null; # be graceful about missing hashes
urls = args.urls or (if args ? url then [ args.url ] else
map (up: "${up}/archive/${urlName}.r${toString revision}.tar.xz") mirrors);
# the basename used by upstream (without ".tar.xz" suffix)
# tlpkg is not a true container but a subfolder of the run container
urlName = pname + (lib.optionalString (tlType != "run" && tlType != "tlpkg") ".${tlType}");
urls = map (up: "${up}/archive/${urlName}.r${toString revision}.tar.xz") mirrors;
# TODO switch to simpler "${name}-${tlOutputName}" (requires new fixed hashes)
container = runCommand "texlive-${pname}${lib.optionalString (tlType != "run") ".${tlType}"}-${version}${extraVersion}"
({
src = fetchurl { inherit urls sha512; };
# save outputName as fixed output derivations cannot change nor override outputName
passthru = passthru // { inherit tlOutputName; };
# TODO remove tlType from derivation (requires a rebuild)
inherit meta stripPrefix tlType;
} // lib.optionalAttrs (fixedHash != null) {
outputHash = fixedHash;
outputHashAlgo = "sha256";
outputHashMode = "recursive";
})
(''
mkdir "$out"
if [[ "$tlType" == "tlpkg" ]]; then
tar -xf "$src" \
--strip-components=1 \
-C "$out" --anchored --exclude=tlpkg/tlpobj --keep-old-files \
tlpkg
else
tar -xf "$src" \
--strip-components="$stripPrefix" \
-C "$out" --anchored --exclude=tlpkg --keep-old-files
fi
'' + postUnpack);
in
runCommand "texlive-${tlName}"
({
src = fetchurl { inherit urls sha512; };
inherit meta passthru stripPrefix tlType;
} // lib.optionalAttrs (fixedHash != null) {
outputHash = fixedHash;
outputHashAlgo = "sha256";
outputHashMode = "recursive";
})
(''
mkdir "$out"
if [[ "$tlType" == "tlpkg" ]]; then
tar -xf "$src" \
--strip-components=1 \
-C "$out" --anchored --exclude=tlpkg/tlpobj --keep-old-files \
tlpkg
else
tar -xf "$src" \
--strip-components="$stripPrefix" \
-C "$out" --anchored --exclude=tlpkg --keep-old-files
fi
'' + postUnpack);
# remove the standard drv.out, optionally replace it with the bin container
builtins.removeAttrs container [ "out" ] // lib.optionalAttrs hasBinfiles { out = bin; };
tex = [
(
let passthru = commonPassthru
// lib.optionalAttrs (args ? deps) { tlDeps = args.deps; }
// lib.optionalAttrs (args ? formats) { inherit (args) formats; }
// lib.optionalAttrs hasHyphens { inherit hasHyphens; }; in
if hasRunfiles then mkContainer "run" passthru sha512.run
else (passthru // { tlType = "run"; })
)
];
tex =
if hasRunfiles then mkContainer "run" "tex" sha512.run
else passthru
// { inherit meta; tlOutputName = "tex"; }
// lib.optionalAttrs hasBinfiles { out = bin; };
doc = let passthru = commonPassthru
// lib.optionalAttrs hasManpages { inherit hasManpages; }; in
lib.optional (sha512 ? doc) (mkContainer "doc" passthru sha512.doc);
texdoc = mkContainer "doc" "texdoc" sha512.doc;
source = lib.optional (sha512 ? source) (mkContainer "source" commonPassthru sha512.source);
texsource = mkContainer "source" "texsource" sha512.source;
tlpkg = let passthru = commonPassthru
// lib.optionalAttrs (args ? postactionScript) { postactionScript = args.postactionScript; }; in
lib.optional hasTlpkg (mkContainer "tlpkg" passthru sha512.run);
tlpkg = mkContainer "tlpkg" "tlpkg" sha512.run;
bin = lib.optional (args ? binfiles && args.binfiles != [ ]) (
let
# find interpreters for the script extensions found in tlpdb
extToInput = {
jar = jdk;
lua = texliveBinaries.luatex;
py = python3;
rb = ruby;
sno = snobol4;
tcl = tk;
texlua = texliveBinaries.luatex;
tlu = texliveBinaries.luatex;
};
run = lib.head tex;
in
runCommand "texlive-${pname}.bin-${version}"
{
passthru = commonPassthru // { tlType = "bin"; };
inherit meta;
# shebang interpreters and compiled binaries
buildInputs = let outName = builtins.replaceStrings [ "-" ] [ "_" ] pname; in
[ texliveBinaries.core.${outName} or null
texliveBinaries.${pname} or null
texliveBinaries.core-big.${outName} or null ]
++ (args.extraBuildInputs or [ ]) ++ [ bash perl ]
++ (lib.attrVals (args.scriptExts or [ ]) extToInput);
nativeBuildInputs = extraNativeBuildInputs;
# absolute scripts folder
scriptsFolder = lib.optionalString (run ? outPath) (run.outPath + "/scripts/" + args.scriptsFolder or pname);
# binaries info
inherit (args) binfiles;
binlinks = builtins.attrNames (args.binlinks or { });
bintargets = builtins.attrValues (args.binlinks or { });
# build scripts
patchScripts = ./patch-scripts.sed;
makeBinContainers = ./make-bin-containers.sh;
}
''
. "$makeBinContainers"
${args.postFixup or ""}
''
);
# build bin container
extToInput = {
# find interpreters for the script extensions found in tlpdb
jar = jdk;
lua = texliveBinaries.luatex;
py = python3;
rb = ruby;
sno = snobol4;
tcl = tk;
texlua = texliveBinaries.luatex;
tlu = texliveBinaries.luatex;
};
# TODO switch to simpler "${name}" (requires a rebuild)
bin = runCommand "texlive-${pname}.bin-${version}"
{
inherit meta;
passthru = passthru // { tlOutputName = "out"; };
# shebang interpreters
buildInputs =let outName = builtins.replaceStrings [ "-" ] [ "_" ] pname; in
[ texliveBinaries.core.${outName} or null
texliveBinaries.${pname} or null
texliveBinaries.core-big.${outName} or null ]
++ (args.extraBuildInputs or [ ]) ++ [ bash perl ]
++ (lib.attrVals (args.scriptExts or [ ]) extToInput);
nativeBuildInputs = extraNativeBuildInputs;
# absolute scripts folder
scriptsFolder = lib.optionalString (tex ? outPath) (tex.outPath + "/scripts/" + args.scriptsFolder or pname);
# binaries info
inherit (args) binfiles;
binlinks = builtins.attrNames (args.binlinks or { });
bintargets = builtins.attrValues (args.binlinks or { });
# build scripts
patchScripts = ./patch-scripts.sed;
makeBinContainers = ./make-bin-containers.sh;
}
''
. "$makeBinContainers"
${args.postFixup or ""}
'';
# build man, info containers
# TODO switch to simpler "${name}-man" (requires a rebuild)
man = builtins.removeAttrs (runCommand "texlive-${pname}.man-${version}${extraVersion}"
{
inherit meta texdoc;
passthru = passthru // { tlOutputName = "man"; };
}
''
mkdir -p "$out"/share
ln -s {"$texdoc"/doc,"$out"/share}/man
'') [ "out" ] // lib.optionalAttrs hasBinfiles { out = bin; };
# TODO switch to simpler "${name}-info" (requires a rebuild)
info = builtins.removeAttrs (runCommand "texlive-${pname}.info-${version}${extraVersion}"
{
inherit meta texdoc;
passthru = passthru // { tlOutputName = "info"; };
}
''
mkdir -p "$out"/share
ln -s {"$texdoc"/doc,"$out"/share}/info
'') [ "out" ] // lib.optionalAttrs hasBinfiles { out = bin; };
in
{ pkgs = tex ++ doc ++ source ++ tlpkg ++ bin; }
builtins.removeAttrs mainDrv [ "outputSpecified" ]

View file

@ -1,6 +1,6 @@
{ lib, buildEnv, runCommand, writeText, makeWrapper, libfaketime, makeFontsConf
, perl, bash, coreutils, gnused, gnugrep, gawk, ghostscript
, bin, tl }:
, bin, tl, toTLPkgList }:
# combine =
args@{
pkgFilter ? (pkg: pkg.tlType == "run" || pkg.tlType == "bin" || pkg.pname == "core"
@ -15,11 +15,11 @@ let
let
# a TeX package is an attribute set { pkgs = [ ... ]; ... } where pkgs is a list of derivations
# the derivations make up the TeX package and optionally (for backward compatibility) its dependencies
tlPkgToSets = { pkgs, ... }: map ({ tlType, version ? "", outputName ? "", ... }@pkg: {
tlPkgToSets = drv: map ({ tlType, version ? "", outputName ? "", ... }@pkg: {
# outputName required to distinguish among bin.core-big outputs
key = "${pkg.pname or pkg.name}.${tlType}-${version}-${outputName}";
inherit pkg;
}) pkgs;
}) (drv.pkgs or (toTLPkgList drv));
pkgListToSets = lib.concatMap tlPkgToSets; in
builtins.genericClosure {
startSet = pkgListToSets pkgList;
@ -49,7 +49,7 @@ let
paths = lib.catAttrs "outPath" pkgList.nonbin;
# mktexlsr
nativeBuildInputs = [ (lib.last tl."texlive.infra".pkgs) ];
nativeBuildInputs = [ tl."texlive.infra" ];
postBuild = # generate ls-R database
''
@ -107,9 +107,9 @@ in (buildEnv {
nativeBuildInputs = [
makeWrapper
libfaketime
(lib.last tl."texlive.infra".pkgs) # mktexlsr
(lib.last tl.texlive-scripts.pkgs) # fmtutil, updmap
(lib.last tl.texlive-scripts-extra.pkgs) # texlinks
tl."texlive.infra" # mktexlsr
tl.texlive-scripts # fmtutil, updmap
tl.texlive-scripts-extra # texlinks
perl
];

View file

@ -24,7 +24,7 @@ let
# function for creating a working environment from a set of TL packages
combine = import ./combine.nix {
inherit bin buildEnv lib makeWrapper writeText runCommand
inherit bin buildEnv lib makeWrapper writeText runCommand toTLPkgList
perl libfaketime makeFontsConf bash tl coreutils gawk gnugrep gnused;
ghostscript = ghostscript_headless;
};
@ -101,12 +101,29 @@ let
// lib.optionalAttrs (args ? deps) { deps = map (n: tl.${n}) (args.deps or [ ]); })
) overriddenTlpdb;
### texlive.combine compatibility layer:
# convert TeX packages to { pkgs = [ ... ]; } lists
# respecting specified outputs
toTLPkgList = drv: if drv.outputSpecified or false
then let tlType = drv.tlType or tlOutToType.${drv.tlOutputName or drv.outputName} or null; in
lib.optional (tlType != null) (drv // { inherit tlType; })
else [ (drv.tex // { tlType = "run"; }) ] ++
lib.optional (drv ? texdoc) (drv.texdoc // { tlType = "doc"; } // lib.optionalAttrs (drv ? man) { hasManpages = true; }) ++
lib.optional (drv ? texsource) (drv.texsource // { tlType = "source"; }) ++
lib.optional (drv ? tlpkg) (drv.tlpkg // { tlType = "tlpkg"; }) ++
lib.optional (drv ? out) (drv.out // { tlType = "bin"; });
tlOutToType = { out = "bin"; tex = "run"; texsource = "source"; texdoc = "doc"; tlpkg = "tlpkg"; };
# export TeX packages as { pkgs = [ ... ]; } in the top attribute set
allPkgLists = lib.mapAttrs (n: drv: { pkgs = toTLPkgList drv; }) tl;
assertions = with lib;
assertMsg (tlpdbVersion.year == version.texliveYear) "TeX Live year in texlive does not match tlpdb.nix, refusing to evaluate" &&
assertMsg (tlpdbVersion.frozen == version.final) "TeX Live final status in texlive does not match tlpdb.nix, refusing to evaluate";
in
tl // {
allPkgLists // {
pkgs = tl;
tlpdb = {
# nested in an attribute set to prevent them from appearing in search
@ -116,7 +133,7 @@ in
bin = assert assertions; bin // {
# for backward compatibility
latexindent = lib.findFirst (p: p.tlType == "bin") tl.latexindent.pkgs;
latexindent = tl.latexindent;
};
combine = assert assertions; combine;

View file

@ -1,13 +1,13 @@
with import ../../../../.. { };
with lib; let
isFod = p: p.tlType != "bin" && isDerivation p;
getFods = drv: lib.optional (isDerivation drv.tex) (drv.tex // { tlType = "run"; })
++ lib.optional (drv ? texdoc) (drv.texdoc // { tlType = "doc"; })
++ lib.optional (drv ? texsource) (drv.texsource // { tlType = "source"; })
++ lib.optional (drv ? tlpkg) (drv.tlpkg // { tlType = "tlpkg"; });
# ugly hack to extract combine from collection-latexextra, since it is masked by texlive.combine
combine = lib.findFirst (p: (lib.head p.pkgs).pname == "combine") { pkgs = [ ]; } (lib.head texlive.collection-latexextra.pkgs).tlDeps;
all = filter (p: p ? pkgs) (attrValues (removeAttrs texlive [ "bin" "combine" "combined" "tlpdb" ])) ++ [ combine ];
sorted = sort (a: b: (head a.pkgs).pname < (head b.pkgs).pname) all;
fods = filter isFod (concatMap (p: p.pkgs or [ ]) all);
sorted = sort (a: b: a.pname < b.pname) (attrValues texlive.pkgs);
fods = concatMap getFods sorted;
computeHash = fod: runCommand "${fod.pname}-${fod.tlType}-fixed-hash"
{ buildInputs = [ nix ]; inherit fod; }
@ -15,18 +15,17 @@ with lib; let
hash = fod: fod.outputHash or (builtins.readFile (computeHash fod));
hashes = { pkgs }:
concatMapStrings ({ tlType, ... }@p: lib.optionalString (isFod p) (''${tlType}="${hash p}";'')) pkgs;
hashes = fods:
concatMapStrings ({ tlType, ... }@p: ''${tlType}="${hash p}";'') fods;
hashLine = { pkgs }@pkg:
hashLine = { pname, revision, extraRevision ? "", ... }@drv:
let
fods = lib.filter isFod pkgs;
first = lib.head fods;
fods = getFods drv;
# NOTE: the fixed naming scheme must match default.nix
fixedName = with first; "${pname}-${toString revision}${first.extraRevision or ""}";
fixedName = "${pname}-${toString revision}${extraRevision}";
in
lib.optionalString (fods != [ ]) ''
${strings.escapeNixIdentifier fixedName}={${hashes pkg}};
optionalString (fods != [ ]) ''
${strings.escapeNixIdentifier fixedName}={${hashes fods}};
'';
in
{
@ -37,6 +36,6 @@ in
fixedHashesNix = writeText "fixed-hashes.nix"
''
{
${lib.concatMapStrings hashLine sorted}}
${concatMapStrings hashLine sorted}}
'';
}

View file

@ -126,7 +126,7 @@ in lib.recursiveUpdate orig rec {
texlive-scripts.binlinks = {
mktexfmt = "fmtutil";
texhash = (lib.last tl."texlive.infra".pkgs) + "/bin/mktexlsr";
texhash = tl."texlive.infra" + "/bin/mktexlsr";
};
texlive-scripts-extra.binlinks = {
@ -352,7 +352,7 @@ in lib.recursiveUpdate orig rec {
mkdir -p support/texdoc
touch support/texdoc/NEWS
TEXMFCNF="${lib.head tl.kpathsea.pkgs}/web2c" TEXMF="$out" TEXDOCS=. TEXMFVAR=. \
TEXMFCNF="${tl.kpathsea.tex}/web2c" TEXMF="$out" TEXDOCS=. TEXMFVAR=. \
"${bin.luatex}"/bin/texlua "$out"/scripts/texdoc/texdoc.tlu \
-c texlive_tlpdb=texlive.tlpdb -lM texdoc
@ -362,7 +362,7 @@ in lib.recursiveUpdate orig rec {
# install zsh completion
postFixup = ''
TEXMFCNF="${lib.head tl.kpathsea.pkgs}"/web2c TEXMF="$scriptsFolder/../.." \
TEXMFCNF="${tl.kpathsea.tex}"/web2c TEXMF="$scriptsFolder/../.." \
texlua "$out"/bin/texdoc --print-completion zsh > "$TMPDIR"/_texdoc
substituteInPlace "$TMPDIR"/_texdoc \
--replace 'compdef __texdoc texdoc' '#compdef texdoc' \
@ -381,14 +381,14 @@ in lib.recursiveUpdate orig rec {
license = [ "gpl2Plus" ] ++ lib.toList bin.core.meta.license.shortName ++ orig."texlive.infra".license or [ ];
scriptsFolder = "texlive";
extraBuildInputs = [ coreutils gnused gnupg (lib.last tl.kpathsea.pkgs) (perl.withPackages (ps: with ps; [ Tk ])) ];
extraBuildInputs = [ coreutils gnused gnupg tl.kpathsea (perl.withPackages (ps: with ps; [ Tk ])) ];
# make tlmgr believe it can use kpsewhich to evaluate TEXMFROOT
postFixup = ''
substituteInPlace "$out"/bin/tlmgr \
--replace 'if (-r "$bindir/$kpsewhichname")' 'if (1)'
sed -i '2i$ENV{PATH}='"'"'${lib.makeBinPath [ gnupg ]}'"'"' . ($ENV{PATH} ? ":$ENV{PATH}" : '"'''"');' "$out"/bin/tlmgr
sed -i '2iPATH="${lib.makeBinPath [ coreutils gnused (lib.last tl.kpathsea.pkgs) ]}''${PATH:+:$PATH}"' "$out"/bin/mktexlsr
sed -i '2iPATH="${lib.makeBinPath [ coreutils gnused tl.kpathsea ]}''${PATH:+:$PATH}"' "$out"/bin/mktexlsr
'';
# add minimal texlive.tlpdb