Merge pull request #244835 from zimbatm/data-writers

writers: add data-centric writers
This commit is contained in:
Lassulus 2023-07-25 22:58:15 +02:00 committed by GitHub
commit 17d98b5b7c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 583 additions and 466 deletions

View file

@ -0,0 +1,80 @@
{ lib, runCommandNoCC, dasel }:
let
daselBin = lib.getExe dasel;
inherit (lib)
last
optionalString
types
;
in
rec {
# Creates a transformer function that writes input data to disk, transformed
# by both the `input` and `output` arguments.
#
# Type: makeDataWriter :: input -> output -> nameOrPath -> data -> (any -> string) -> string -> string -> any -> derivation
#
# input :: T -> string: function that takes the nix data and returns a string
# output :: string: script that takes the $inputFile and write the result into $out
# nameOrPath :: string: if the name contains a / the files gets written to a sub-folder of $out. The derivation name is the basename of this argument.
# data :: T: the data that will be converted.
#
# Example:
# writeJSON = makeDataWriter { input = builtins.toJSON; output = "cp $inputPath $out"; };
# myConfig = writeJSON "config.json" { hello = "world"; }
#
makeDataWriter = { input ? lib.id, output ? "cp $inputPath $out" }: nameOrPath: data:
assert lib.or (types.path.check nameOrPath) (builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null);
let
name = last (builtins.split "/" nameOrPath);
in
runCommandNoCC name
{
input = input data;
passAsFile = [ "input" ];
} ''
${output}
${optionalString (types.path.check nameOrPath) ''
mv $out tmp
mkdir -p $out/$(dirname "${nameOrPath}")
mv tmp $out/${nameOrPath}
''}
'';
# Writes the content to text.
#
# Example:
# writeText "filename.txt" "file content"
writeText = makeDataWriter {
input = toString;
output = "cp $inputPath $out";
};
# Writes the content to a JSON file.
#
# Example:
# writeJSON "data.json" { hello = "world"; }
writeJSON = makeDataWriter {
input = builtins.toJSON;
output = "${daselBin} -f $inputPath -r json -w json > $out";
};
# Writes the content to a TOML file.
#
# Example:
# writeTOML "data.toml" { hello = "world"; }
writeTOML = makeDataWriter {
input = builtins.toJSON;
output = "${daselBin} -f $inputPath -r json -w toml > $out";
};
# Writes the content to a YAML file.
#
# Example:
# writeYAML "data.yaml" { hello = "world"; }
writeYAML = makeDataWriter {
input = builtins.toJSON;
output = "${daselBin} -f $inputPath -r json -w yaml > $out";
};
}

View file

@ -1,367 +1,18 @@
{ pkgs, config, buildPackages, lib, stdenv, libiconv, mkNugetDeps, mkNugetSource, gixy }:
{ pkgs, config, lib }:
let
aliases = if config.allowAliases then (import ./aliases.nix lib) else prev: {};
writers = with lib; rec {
# Base implementation for non-compiled executables.
# Takes an interpreter, for example `${pkgs.bash}/bin/bash`
#
# Examples:
# writeBash = makeScriptWriter { interpreter = "${pkgs.bash}/bin/bash"; }
# makeScriptWriter { interpreter = "${pkgs.dash}/bin/dash"; } "hello" "echo hello world"
makeScriptWriter = { interpreter, check ? "" }: nameOrPath: content:
assert lib.or (types.path.check nameOrPath) (builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null);
assert lib.or (types.path.check content) (types.str.check content);
let
name = last (builtins.split "/" nameOrPath);
in
pkgs.runCommandLocal name (if (types.str.check content) then {
inherit content interpreter;
passAsFile = [ "content" ];
} else {
inherit interpreter;
contentPath = content;
}) ''
# On darwin a script cannot be used as an interpreter in a shebang but
# there doesn't seem to be a limit to the size of shebang and multiple
# arguments to the interpreter are allowed.
if [[ -n "${toString pkgs.stdenvNoCC.isDarwin}" ]] && isScript $interpreter
then
wrapperInterpreterLine=$(head -1 "$interpreter" | tail -c+3)
# Get first word from the line (note: xargs echo remove leading spaces)
wrapperInterpreter=$(echo "$wrapperInterpreterLine" | xargs echo | cut -d " " -f1)
if isScript $wrapperInterpreter
then
echo "error: passed interpreter ($interpreter) is a script which has another script ($wrapperInterpreter) as an interpreter, which is not supported."
exit 1
fi
# This should work as long as wrapperInterpreter is a shell, which is
# the case for programs wrapped with makeWrapper, like
# python3.withPackages etc.
interpreterLine="$wrapperInterpreterLine $interpreter"
else
interpreterLine=$interpreter
fi
echo "#! $interpreterLine" > $out
cat "$contentPath" >> $out
${optionalString (check != "") ''
${check} $out
''}
chmod +x $out
${optionalString (types.path.check nameOrPath) ''
mv $out tmp
mkdir -p $out/$(dirname "${nameOrPath}")
mv tmp $out/${nameOrPath}
''}
'';
# Base implementation for compiled executables.
# Takes a compile script, which in turn takes the name as an argument.
#
# Examples:
# writeSimpleC = makeBinWriter { compileScript = name: "gcc -o $out $contentPath"; }
makeBinWriter = { compileScript, strip ? true }: nameOrPath: content:
assert lib.or (types.path.check nameOrPath) (builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null);
assert lib.or (types.path.check content) (types.str.check content);
let
name = last (builtins.split "/" nameOrPath);
in
pkgs.runCommand name ((if (types.str.check content) then {
inherit content;
passAsFile = [ "content" ];
} else {
contentPath = content;
}) // lib.optionalAttrs (stdenv.hostPlatform.isDarwin && stdenv.hostPlatform.isAarch64) {
# post-link-hook expects codesign_allocate to be in PATH
# https://github.com/NixOS/nixpkgs/issues/154203
# https://github.com/NixOS/nixpkgs/issues/148189
nativeBuildInputs = [ stdenv.cc.bintools ];
}) ''
${compileScript}
${lib.optionalString strip
"${lib.getBin buildPackages.bintools-unwrapped}/bin/${buildPackages.bintools-unwrapped.targetPrefix}strip -S $out"}
# Sometimes binaries produced for darwin (e. g. by GHC) won't be valid
# mach-o executables from the get-go, but need to be corrected somehow
# which is done by fixupPhase.
${lib.optionalString pkgs.stdenvNoCC.hostPlatform.isDarwin "fixupPhase"}
${optionalString (types.path.check nameOrPath) ''
mv $out tmp
mkdir -p $out/$(dirname "${nameOrPath}")
mv tmp $out/${nameOrPath}
''}
'';
# Like writeScript but the first line is a shebang to bash
#
# Example:
# writeBash "example" ''
# echo hello world
# ''
writeBash = makeScriptWriter {
interpreter = "${pkgs.bash}/bin/bash";
# Writers for JSON-like data structures
dataWriters = import ./data.nix {
inherit lib; inherit (pkgs) runCommandNoCC dasel;
};
# Like writeScriptBin but the first line is a shebang to bash
writeBashBin = name:
writeBash "/bin/${name}";
# Like writeScript but the first line is a shebang to dash
#
# Example:
# writeDash "example" ''
# echo hello world
# ''
writeDash = makeScriptWriter {
interpreter = "${pkgs.dash}/bin/dash";
# Writers for scripts
scriptWriters = import ./scripts.nix {
inherit lib pkgs;
};
# Like writeScriptBin but the first line is a shebang to dash
writeDashBin = name:
writeDash "/bin/${name}";
# Like writeScript but the first line is a shebang to fish
#
# Example:
# writeFish "example" ''
# echo hello world
# ''
writeFish = makeScriptWriter {
interpreter = "${pkgs.fish}/bin/fish --no-config";
check = "${pkgs.fish}/bin/fish --no-config --no-execute"; # syntax check only
};
# Like writeScriptBin but the first line is a shebang to fish
writeFishBin = name:
writeFish "/bin/${name}";
# writeHaskell takes a name, an attrset with libraries and haskell version (both optional)
# and some haskell source code and returns an executable.
#
# Example:
# writeHaskell "missiles" { libraries = [ pkgs.haskellPackages.acme-missiles ]; } ''
# import Acme.Missiles
#
# main = launchMissiles
# '';
writeHaskell = name: {
libraries ? [],
ghc ? pkgs.ghc,
ghcArgs ? [],
threadedRuntime ? true,
strip ? true
}:
let
appendIfNotSet = el: list: if elem el list then list else list ++ [ el ];
ghcArgs' = if threadedRuntime then appendIfNotSet "-threaded" ghcArgs else ghcArgs;
in makeBinWriter {
compileScript = ''
cp $contentPath tmp.hs
${ghc.withPackages (_: libraries )}/bin/ghc ${lib.escapeShellArgs ghcArgs'} tmp.hs
mv tmp $out
'';
inherit strip;
} name;
# writeHaskellBin takes the same arguments as writeHaskell but outputs a directory (like writeScriptBin)
writeHaskellBin = name:
writeHaskell "/bin/${name}";
writeRust = name: {
rustc ? pkgs.rustc,
rustcArgs ? [],
strip ? true
}:
let
darwinArgs = lib.optionals stdenv.isDarwin [ "-L${lib.getLib libiconv}/lib" ];
in
makeBinWriter {
compileScript = ''
cp "$contentPath" tmp.rs
PATH=${makeBinPath [pkgs.gcc]} ${lib.getBin rustc}/bin/rustc ${lib.escapeShellArgs rustcArgs} ${lib.escapeShellArgs darwinArgs} -o "$out" tmp.rs
'';
inherit strip;
} name;
writeRustBin = name:
writeRust "/bin/${name}";
# writeJS takes a name an attributeset with libraries and some JavaScript sourcecode and
# returns an executable
#
# Example:
# writeJS "example" { libraries = [ pkgs.nodePackages.uglify-js ]; } ''
# var UglifyJS = require("uglify-js");
# var code = "function add(first, second) { return first + second; }";
# var result = UglifyJS.minify(code);
# console.log(result.code);
# ''
writeJS = name: { libraries ? [] }: content:
let
node-env = pkgs.buildEnv {
name = "node";
paths = libraries;
pathsToLink = [
"/lib/node_modules"
];
};
in writeDash name ''
export NODE_PATH=${node-env}/lib/node_modules
exec ${pkgs.nodejs}/bin/node ${pkgs.writeText "js" content} "$@"
'';
# writeJSBin takes the same arguments as writeJS but outputs a directory (like writeScriptBin)
writeJSBin = name:
writeJS "/bin/${name}";
awkFormatNginx = builtins.toFile "awkFormat-nginx.awk" ''
awk -f
{sub(/^[ \t]+/,"");idx=0}
/\{/{ctx++;idx=1}
/\}/{ctx--}
{id="";for(i=idx;i<ctx;i++)id=sprintf("%s%s", id, "\t");printf "%s%s\n", id, $0}
'';
writeNginxConfig = name: text: pkgs.runCommandLocal name {
inherit text;
passAsFile = [ "text" ];
nativeBuildInputs = [ gixy ];
} /* sh */ ''
# nginx-config-formatter has an error - https://github.com/1connect/nginx-config-formatter/issues/16
awk -f ${awkFormatNginx} "$textPath" | sed '/^\s*$/d' > $out
gixy $out
'';
# writePerl takes a name an attributeset with libraries and some perl sourcecode and
# returns an executable
#
# Example:
# writePerl "example" { libraries = [ pkgs.perlPackages.boolean ]; } ''
# use boolean;
# print "Howdy!\n" if true;
# ''
writePerl = name: { libraries ? [] }:
makeScriptWriter {
interpreter = "${pkgs.perl.withPackages (p: libraries)}/bin/perl";
} name;
# writePerlBin takes the same arguments as writePerl but outputs a directory (like writeScriptBin)
writePerlBin = name:
writePerl "/bin/${name}";
# makePythonWriter takes python and compatible pythonPackages and produces python script writer,
# which validates the script with flake8 at build time. If any libraries are specified,
# python.withPackages is used as interpreter, otherwise the "bare" python is used.
makePythonWriter = python: pythonPackages: buildPythonPackages: name: { libraries ? [], flakeIgnore ? [] }:
let
ignoreAttribute = optionalString (flakeIgnore != []) "--ignore ${concatMapStringsSep "," escapeShellArg flakeIgnore}";
in
makeScriptWriter {
interpreter =
if libraries == []
then python.interpreter
else (python.withPackages (ps: libraries)).interpreter
;
check = optionalString python.isPy3k (writeDash "pythoncheck.sh" ''
exec ${buildPythonPackages.flake8}/bin/flake8 --show-source ${ignoreAttribute} "$1"
'');
} name;
# writePyPy2 takes a name an attributeset with libraries and some pypy2 sourcecode and
# returns an executable
#
# Example:
# writePyPy2 "test_pypy2" { libraries = [ pkgs.pypy2Packages.enum ]; } ''
# from enum import Enum
#
# class Test(Enum):
# a = "success"
#
# print Test.a
# ''
writePyPy2 = makePythonWriter pkgs.pypy2 pkgs.pypy2Packages buildPackages.pypy2Packages;
# writePyPy2Bin takes the same arguments as writePyPy2 but outputs a directory (like writeScriptBin)
writePyPy2Bin = name:
writePyPy2 "/bin/${name}";
# writePython3 takes a name an attributeset with libraries and some python3 sourcecode and
# returns an executable
#
# Example:
# writePython3 "test_python3" { libraries = [ pkgs.python3Packages.pyyaml ]; } ''
# import yaml
#
# y = yaml.load("""
# - test: success
# """)
# print(y[0]['test'])
# ''
writePython3 = makePythonWriter pkgs.python3 pkgs.python3Packages buildPackages.python3Packages;
# writePython3Bin takes the same arguments as writePython3 but outputs a directory (like writeScriptBin)
writePython3Bin = name:
writePython3 "/bin/${name}";
# writePyPy3 takes a name an attributeset with libraries and some pypy3 sourcecode and
# returns an executable
#
# Example:
# writePyPy3 "test_pypy3" { libraries = [ pkgs.pypy3Packages.pyyaml ]; } ''
# import yaml
#
# y = yaml.load("""
# - test: success
# """)
# print(y[0]['test'])
# ''
writePyPy3 = makePythonWriter pkgs.pypy3 pkgs.pypy3Packages buildPackages.pypy3Packages;
# writePyPy3Bin takes the same arguments as writePyPy3 but outputs a directory (like writeScriptBin)
writePyPy3Bin = name:
writePyPy3 "/bin/${name}";
makeFSharpWriter = { dotnet-sdk ? pkgs.dotnet-sdk, fsi-flags ? "", libraries ? _: [] }: nameOrPath:
let
fname = last (builtins.split "/" nameOrPath);
path = if strings.hasSuffix ".fsx" nameOrPath then nameOrPath else "${nameOrPath}.fsx";
_nugetDeps = mkNugetDeps { name = "${fname}-nuget-deps"; nugetDeps = libraries; };
nuget-source = mkNugetSource {
name = "${fname}-nuget-source";
description = "A Nuget source with the dependencies for ${fname}";
deps = [ _nugetDeps ];
};
fsi = writeBash "fsi" ''
export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
export DOTNET_CLI_TELEMETRY_OPTOUT=1
export DOTNET_NOLOGO=1
script="$1"; shift
${dotnet-sdk}/bin/dotnet fsi --quiet --nologo --readline- ${fsi-flags} "$@" < "$script"
'';
in content: writers.makeScriptWriter {
interpreter = fsi;
} path
''
#i "nuget: ${nuget-source}/lib"
${ content }
exit 0
'';
writeFSharp =
makeFSharpWriter {};
writeFSharpBin = name:
writeFSharp "/bin/${name}";
};
writers = scriptWriters // dataWriters;
in
writers // (aliases writers)

View file

@ -0,0 +1,383 @@
{ pkgs, lib }:
let
inherit (lib)
concatMapStringsSep
elem
escapeShellArg
last
optionalString
stringLength
strings
types
;
inherit (pkgs)
buildPackages
gixy
libiconv
mkNugetDeps
mkNugetSource
stdenv
;
in
rec {
# Base implementation for non-compiled executables.
# Takes an interpreter, for example `${pkgs.bash}/bin/bash`
#
# Examples:
# writeBash = makeScriptWriter { interpreter = "${pkgs.bash}/bin/bash"; }
# makeScriptWriter { interpreter = "${pkgs.dash}/bin/dash"; } "hello" "echo hello world"
makeScriptWriter = { interpreter, check ? "" }: nameOrPath: content:
assert lib.or (types.path.check nameOrPath) (builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null);
assert lib.or (types.path.check content) (types.str.check content);
let
name = last (builtins.split "/" nameOrPath);
in
pkgs.runCommandLocal name (if (types.str.check content) then {
inherit content interpreter;
passAsFile = [ "content" ];
} else {
inherit interpreter;
contentPath = content;
}) ''
# On darwin a script cannot be used as an interpreter in a shebang but
# there doesn't seem to be a limit to the size of shebang and multiple
# arguments to the interpreter are allowed.
if [[ -n "${toString pkgs.stdenvNoCC.isDarwin}" ]] && isScript $interpreter
then
wrapperInterpreterLine=$(head -1 "$interpreter" | tail -c+3)
# Get first word from the line (note: xargs echo remove leading spaces)
wrapperInterpreter=$(echo "$wrapperInterpreterLine" | xargs echo | cut -d " " -f1)
if isScript $wrapperInterpreter
then
echo "error: passed interpreter ($interpreter) is a script which has another script ($wrapperInterpreter) as an interpreter, which is not supported."
exit 1
fi
# This should work as long as wrapperInterpreter is a shell, which is
# the case for programs wrapped with makeWrapper, like
# python3.withPackages etc.
interpreterLine="$wrapperInterpreterLine $interpreter"
else
interpreterLine=$interpreter
fi
echo "#! $interpreterLine" > $out
cat "$contentPath" >> $out
${optionalString (check != "") ''
${check} $out
''}
chmod +x $out
${optionalString (types.path.check nameOrPath) ''
mv $out tmp
mkdir -p $out/$(dirname "${nameOrPath}")
mv tmp $out/${nameOrPath}
''}
'';
# Base implementation for compiled executables.
# Takes a compile script, which in turn takes the name as an argument.
#
# Examples:
# writeSimpleC = makeBinWriter { compileScript = name: "gcc -o $out $contentPath"; }
makeBinWriter = { compileScript, strip ? true }: nameOrPath: content:
assert lib.or (types.path.check nameOrPath) (builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null);
assert lib.or (types.path.check content) (types.str.check content);
let
name = last (builtins.split "/" nameOrPath);
in
pkgs.runCommand name ((if (types.str.check content) then {
inherit content;
passAsFile = [ "content" ];
} else {
contentPath = content;
}) // lib.optionalAttrs (stdenv.hostPlatform.isDarwin && stdenv.hostPlatform.isAarch64) {
# post-link-hook expects codesign_allocate to be in PATH
# https://github.com/NixOS/nixpkgs/issues/154203
# https://github.com/NixOS/nixpkgs/issues/148189
nativeBuildInputs = [ stdenv.cc.bintools ];
}) ''
${compileScript}
${lib.optionalString strip
"${lib.getBin buildPackages.bintools-unwrapped}/bin/${buildPackages.bintools-unwrapped.targetPrefix}strip -S $out"}
# Sometimes binaries produced for darwin (e. g. by GHC) won't be valid
# mach-o executables from the get-go, but need to be corrected somehow
# which is done by fixupPhase.
${lib.optionalString pkgs.stdenvNoCC.hostPlatform.isDarwin "fixupPhase"}
${optionalString (types.path.check nameOrPath) ''
mv $out tmp
mkdir -p $out/$(dirname "${nameOrPath}")
mv tmp $out/${nameOrPath}
''}
'';
# Like writeScript but the first line is a shebang to bash
#
# Example:
# writeBash "example" ''
# echo hello world
# ''
writeBash = makeScriptWriter {
interpreter = "${pkgs.bash}/bin/bash";
};
# Like writeScriptBin but the first line is a shebang to bash
writeBashBin = name:
writeBash "/bin/${name}";
# Like writeScript but the first line is a shebang to dash
#
# Example:
# writeDash "example" ''
# echo hello world
# ''
writeDash = makeScriptWriter {
interpreter = "${pkgs.dash}/bin/dash";
};
# Like writeScriptBin but the first line is a shebang to dash
writeDashBin = name:
writeDash "/bin/${name}";
# Like writeScript but the first line is a shebang to fish
#
# Example:
# writeFish "example" ''
# echo hello world
# ''
writeFish = makeScriptWriter {
interpreter = "${pkgs.fish}/bin/fish --no-config";
check = "${pkgs.fish}/bin/fish --no-config --no-execute"; # syntax check only
};
# Like writeScriptBin but the first line is a shebang to fish
writeFishBin = name:
writeFish "/bin/${name}";
# writeHaskell takes a name, an attrset with libraries and haskell version (both optional)
# and some haskell source code and returns an executable.
#
# Example:
# writeHaskell "missiles" { libraries = [ pkgs.haskellPackages.acme-missiles ]; } ''
# import Acme.Missiles
#
# main = launchMissiles
# '';
writeHaskell = name: {
libraries ? [],
ghc ? pkgs.ghc,
ghcArgs ? [],
threadedRuntime ? true,
strip ? true
}:
let
appendIfNotSet = el: list: if elem el list then list else list ++ [ el ];
ghcArgs' = if threadedRuntime then appendIfNotSet "-threaded" ghcArgs else ghcArgs;
in makeBinWriter {
compileScript = ''
cp $contentPath tmp.hs
${ghc.withPackages (_: libraries )}/bin/ghc ${lib.escapeShellArgs ghcArgs'} tmp.hs
mv tmp $out
'';
inherit strip;
} name;
# writeHaskellBin takes the same arguments as writeHaskell but outputs a directory (like writeScriptBin)
writeHaskellBin = name:
writeHaskell "/bin/${name}";
writeRust = name: {
rustc ? pkgs.rustc,
rustcArgs ? [],
strip ? true
}:
let
darwinArgs = lib.optionals stdenv.isDarwin [ "-L${lib.getLib libiconv}/lib" ];
in
makeBinWriter {
compileScript = ''
cp "$contentPath" tmp.rs
PATH=${lib.makeBinPath [pkgs.gcc]} ${lib.getBin rustc}/bin/rustc ${lib.escapeShellArgs rustcArgs} ${lib.escapeShellArgs darwinArgs} -o "$out" tmp.rs
'';
inherit strip;
} name;
writeRustBin = name:
writeRust "/bin/${name}";
# writeJS takes a name an attributeset with libraries and some JavaScript sourcecode and
# returns an executable
#
# Example:
# writeJS "example" { libraries = [ pkgs.nodePackages.uglify-js ]; } ''
# var UglifyJS = require("uglify-js");
# var code = "function add(first, second) { return first + second; }";
# var result = UglifyJS.minify(code);
# console.log(result.code);
# ''
writeJS = name: { libraries ? [] }: content:
let
node-env = pkgs.buildEnv {
name = "node";
paths = libraries;
pathsToLink = [
"/lib/node_modules"
];
};
in writeDash name ''
export NODE_PATH=${node-env}/lib/node_modules
exec ${pkgs.nodejs}/bin/node ${pkgs.writeText "js" content} "$@"
'';
# writeJSBin takes the same arguments as writeJS but outputs a directory (like writeScriptBin)
writeJSBin = name:
writeJS "/bin/${name}";
awkFormatNginx = builtins.toFile "awkFormat-nginx.awk" ''
awk -f
{sub(/^[ \t]+/,"");idx=0}
/\{/{ctx++;idx=1}
/\}/{ctx--}
{id="";for(i=idx;i<ctx;i++)id=sprintf("%s%s", id, "\t");printf "%s%s\n", id, $0}
'';
writeNginxConfig = name: text: pkgs.runCommandLocal name {
inherit text;
passAsFile = [ "text" ];
nativeBuildInputs = [ gixy ];
} /* sh */ ''
# nginx-config-formatter has an error - https://github.com/1connect/nginx-config-formatter/issues/16
awk -f ${awkFormatNginx} "$textPath" | sed '/^\s*$/d' > $out
gixy $out
'';
# writePerl takes a name an attributeset with libraries and some perl sourcecode and
# returns an executable
#
# Example:
# writePerl "example" { libraries = [ pkgs.perlPackages.boolean ]; } ''
# use boolean;
# print "Howdy!\n" if true;
# ''
writePerl = name: { libraries ? [] }:
makeScriptWriter {
interpreter = "${pkgs.perl.withPackages (p: libraries)}/bin/perl";
} name;
# writePerlBin takes the same arguments as writePerl but outputs a directory (like writeScriptBin)
writePerlBin = name:
writePerl "/bin/${name}";
# makePythonWriter takes python and compatible pythonPackages and produces python script writer,
# which validates the script with flake8 at build time. If any libraries are specified,
# python.withPackages is used as interpreter, otherwise the "bare" python is used.
makePythonWriter = python: pythonPackages: buildPythonPackages: name: { libraries ? [], flakeIgnore ? [] }:
let
ignoreAttribute = optionalString (flakeIgnore != []) "--ignore ${concatMapStringsSep "," escapeShellArg flakeIgnore}";
in
makeScriptWriter {
interpreter =
if libraries == []
then python.interpreter
else (python.withPackages (ps: libraries)).interpreter
;
check = optionalString python.isPy3k (writeDash "pythoncheck.sh" ''
exec ${buildPythonPackages.flake8}/bin/flake8 --show-source ${ignoreAttribute} "$1"
'');
} name;
# writePyPy2 takes a name an attributeset with libraries and some pypy2 sourcecode and
# returns an executable
#
# Example:
# writePyPy2 "test_pypy2" { libraries = [ pkgs.pypy2Packages.enum ]; } ''
# from enum import Enum
#
# class Test(Enum):
# a = "success"
#
# print Test.a
# ''
writePyPy2 = makePythonWriter pkgs.pypy2 pkgs.pypy2Packages buildPackages.pypy2Packages;
# writePyPy2Bin takes the same arguments as writePyPy2 but outputs a directory (like writeScriptBin)
writePyPy2Bin = name:
writePyPy2 "/bin/${name}";
# writePython3 takes a name an attributeset with libraries and some python3 sourcecode and
# returns an executable
#
# Example:
# writePython3 "test_python3" { libraries = [ pkgs.python3Packages.pyyaml ]; } ''
# import yaml
#
# y = yaml.load("""
# - test: success
# """)
# print(y[0]['test'])
# ''
writePython3 = makePythonWriter pkgs.python3 pkgs.python3Packages buildPackages.python3Packages;
# writePython3Bin takes the same arguments as writePython3 but outputs a directory (like writeScriptBin)
writePython3Bin = name:
writePython3 "/bin/${name}";
# writePyPy3 takes a name an attributeset with libraries and some pypy3 sourcecode and
# returns an executable
#
# Example:
# writePyPy3 "test_pypy3" { libraries = [ pkgs.pypy3Packages.pyyaml ]; } ''
# import yaml
#
# y = yaml.load("""
# - test: success
# """)
# print(y[0]['test'])
# ''
writePyPy3 = makePythonWriter pkgs.pypy3 pkgs.pypy3Packages buildPackages.pypy3Packages;
# writePyPy3Bin takes the same arguments as writePyPy3 but outputs a directory (like writeScriptBin)
writePyPy3Bin = name:
writePyPy3 "/bin/${name}";
makeFSharpWriter = { dotnet-sdk ? pkgs.dotnet-sdk, fsi-flags ? "", libraries ? _: [] }: nameOrPath:
let
fname = last (builtins.split "/" nameOrPath);
path = if strings.hasSuffix ".fsx" nameOrPath then nameOrPath else "${nameOrPath}.fsx";
_nugetDeps = mkNugetDeps { name = "${fname}-nuget-deps"; nugetDeps = libraries; };
nuget-source = mkNugetSource {
name = "${fname}-nuget-source";
description = "A Nuget source with the dependencies for ${fname}";
deps = [ _nugetDeps ];
};
fsi = writeBash "fsi" ''
export HOME=$NIX_BUILD_TOP/.home
export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
export DOTNET_CLI_TELEMETRY_OPTOUT=1
export DOTNET_NOLOGO=1
script="$1"; shift
${dotnet-sdk}/bin/dotnet fsi --quiet --nologo --readline- ${fsi-flags} "$@" < "$script"
'';
in content: makeScriptWriter {
interpreter = fsi;
} path
''
#i "nuget: ${nuget-source}/lib"
${ content }
exit 0
'';
writeFSharp =
makeFSharpWriter {};
writeFSharpBin = name:
writeFSharp "/bin/${name}";
}

View file

@ -12,29 +12,62 @@
}:
with writers;
let
expectSuccess = test:
runCommand "run-${test.name}" {} ''
if [[ "$(${test})" != success ]]; then
echo 'test ${test.name} failed'
exit 1
fi
bin = {
bash = writeBashBin "test-writers-bash-bin" ''
touch $out
'';
expectSuccessBin = test:
runCommand "run-${test.name}" {} ''
if [[ "$(${lib.getExe test})" != success ]]; then
echo 'test ${test.name} failed'
exit 1
fi
touch $out
'';
expectDataEqual = { file, expected }:
let
expectedFile = writeText "${file.name}-expected" expected;
in
runCommand "run-${file.name}" {} ''
if ! diff -u ${file} ${expectedFile}; then
echo 'test ${file.name} failed'
exit 1
fi
touch $out
'';
in
lib.recurseIntoAttrs {
bin = lib.recurseIntoAttrs {
bash = expectSuccessBin (writeBashBin "test-writers-bash-bin" ''
if [[ "test" == "test" ]]; then echo "success"; fi
'';
'');
dash = writeDashBin "test-writers-dash-bin" ''
dash = expectSuccessBin (writeDashBin "test-writers-dash-bin" ''
test '~' = '~' && echo 'success'
'';
'');
fish = writeFishBin "test-writers-fish-bin" ''
fish = expectSuccessBin (writeFishBin "test-writers-fish-bin" ''
if test "test" = "test"
echo "success"
end
'';
'');
rust = writeRustBin "test-writers-rust-bin" {} ''
rust = expectSuccessBin (writeRustBin "test-writers-rust-bin" {} ''
fn main(){
println!("success")
}
'';
'');
haskell = writeHaskellBin "test-writers-haskell-bin" { libraries = [ haskellPackages.acme-default ]; } ''
haskell = expectSuccessBin (writeHaskellBin "test-writers-haskell-bin" { libraries = [ haskellPackages.acme-default ]; } ''
import Data.Default
int :: Int
@ -44,9 +77,9 @@ let
main = case int of
18871 -> putStrLn $ id "success"
_ -> print "fail"
'';
'');
js = writeJSBin "test-writers-js-bin" { libraries = [ nodePackages.semver ]; } ''
js = expectSuccessBin (writeJSBin "test-writers-js-bin" { libraries = [ nodePackages.semver ]; } ''
var semver = require('semver');
if (semver.valid('1.2.3')) {
@ -54,59 +87,57 @@ let
} else {
console.log('fail')
}
'';
'');
perl = writePerlBin "test-writers-perl-bin" { libraries = [ perlPackages.boolean ]; } ''
perl = expectSuccessBin (writePerlBin "test-writers-perl-bin" { libraries = [ perlPackages.boolean ]; } ''
use boolean;
print "success\n" if true;
'';
'');
pypy2 = writePyPy2Bin "test-writers-pypy2-bin" { libraries = [ pypy2Packages.enum ]; } ''
pypy2 = expectSuccessBin (writePyPy2Bin "test-writers-pypy2-bin" { libraries = [ pypy2Packages.enum ]; } ''
from enum import Enum
class Test(Enum):
a = "success"
print Test.a
'';
'');
python3 = writePython3Bin "test-writers-python3-bin" { libraries = [ python3Packages.pyyaml ]; } ''
python3 = expectSuccessBin (writePython3Bin "test-writers-python3-bin" { libraries = [ python3Packages.pyyaml ]; } ''
import yaml
y = yaml.load("""
y = yaml.safe_load("""
- test: success
""")
print(y[0]['test'])
'';
'');
pypy3 = writePyPy3Bin "test-writers-pypy3-bin" { libraries = [ pypy3Packages.pyyaml ]; } ''
pypy3 = expectSuccessBin (writePyPy3Bin "test-writers-pypy3-bin" { libraries = [ pypy3Packages.pyyaml ]; } ''
import yaml
y = yaml.load("""
y = yaml.safe_load("""
- test: success
""")
print(y[0]['test'])
'';
'');
};
simple = {
bash = writeBash "test-writers-bash" ''
simple = lib.recurseIntoAttrs {
bash = expectSuccess (writeBash "test-writers-bash" ''
if [[ "test" == "test" ]]; then echo "success"; fi
'';
'');
dash = writeDash "test-writers-dash" ''
dash = expectSuccess (writeDash "test-writers-dash" ''
test '~' = '~' && echo 'success'
'';
'');
fish = writeFish "test-writers-fish" ''
fish = expectSuccess (writeFish "test-writers-fish" ''
if test "test" = "test"
echo "success"
end
'';
'');
haskell = writeHaskell "test-writers-haskell" { libraries = [ haskellPackages.acme-default ]; } ''
haskell = expectSuccess (writeHaskell "test-writers-haskell" { libraries = [ haskellPackages.acme-default ]; } ''
import Data.Default
int :: Int
@ -116,9 +147,9 @@ let
main = case int of
18871 -> putStrLn $ id "success"
_ -> print "fail"
'';
'');
js = writeJS "test-writers-js" { libraries = [ nodePackages.semver ]; } ''
js = expectSuccess (writeJS "test-writers-js" { libraries = [ nodePackages.semver ]; } ''
var semver = require('semver');
if (semver.valid('1.2.3')) {
@ -126,43 +157,41 @@ let
} else {
console.log('fail')
}
'';
'');
perl = writePerl "test-writers-perl" { libraries = [ perlPackages.boolean ]; } ''
perl = expectSuccess (writePerl "test-writers-perl" { libraries = [ perlPackages.boolean ]; } ''
use boolean;
print "success\n" if true;
'';
'');
pypy2 = writePyPy2 "test-writers-pypy2" { libraries = [ pypy2Packages.enum ]; } ''
pypy2 = expectSuccess (writePyPy2 "test-writers-pypy2" { libraries = [ pypy2Packages.enum ]; } ''
from enum import Enum
class Test(Enum):
a = "success"
print Test.a
'';
'');
python3 = writePython3 "test-writers-python3" { libraries = [ python3Packages.pyyaml ]; } ''
python3 = expectSuccess (writePython3 "test-writers-python3" { libraries = [ python3Packages.pyyaml ]; } ''
import yaml
y = yaml.load("""
y = yaml.safe_load("""
- test: success
""")
print(y[0]['test'])
'';
'');
pypy3 = writePyPy3 "test-writers-pypy3" { libraries = [ pypy3Packages.pyyaml ]; } ''
pypy3 = expectSuccess (writePyPy3 "test-writers-pypy3" { libraries = [ pypy3Packages.pyyaml ]; } ''
import yaml
y = yaml.load("""
y = yaml.safe_load("""
- test: success
""")
print(y[0]['test'])
'';
'');
fsharp = makeFSharpWriter {
fsharp = expectSuccess (makeFSharpWriter {
libraries = { fetchNuGet }: [
(fetchNuGet { pname = "FSharp.SystemTextJson"; version = "0.17.4"; sha256 = "1bplzc9ybdqspii4q28l8gmfvzpkmgq5l1hlsiyg2h46w881lwg2"; })
];
@ -183,31 +212,31 @@ let
then "success"
else "failed"
|> printfn "%s"
'';
'');
pypy2NoLibs = writePyPy2 "test-writers-pypy2-no-libs" {} ''
pypy2NoLibs = expectSuccess (writePyPy2 "test-writers-pypy2-no-libs" {} ''
print("success")
'';
'');
python3NoLibs = writePython3 "test-writers-python3-no-libs" {} ''
python3NoLibs = expectSuccess (writePython3 "test-writers-python3-no-libs" {} ''
print("success")
'';
'');
pypy3NoLibs = writePyPy3 "test-writers-pypy3-no-libs" {} ''
pypy3NoLibs = expectSuccess (writePyPy3 "test-writers-pypy3-no-libs" {} ''
print("success")
'';
'');
fsharpNoNugetDeps = writeFSharp "test-writers-fsharp-no-nuget-deps" ''
fsharpNoNugetDeps = expectSuccess (writeFSharp "test-writers-fsharp-no-nuget-deps" ''
printfn "success"
'';
'');
};
path = {
bash = writeBash "test-writers-bash-path" (writeText "test" ''
path = lib.recurseIntoAttrs {
bash = expectSuccess (writeBash "test-writers-bash-path" (writeText "test" ''
if [[ "test" == "test" ]]; then echo "success"; fi
'');
haskell = writeHaskell "test-writers-haskell-path" { libraries = [ haskellPackages.acme-default ]; } (writeText "test" ''
''));
haskell = expectSuccess (writeHaskell "test-writers-haskell-path" { libraries = [ haskellPackages.acme-default ]; } (writeText "test" ''
import Data.Default
int :: Int
@ -217,26 +246,27 @@ let
main = case int of
18871 -> putStrLn $ id "success"
_ -> print "fail"
'');
''));
};
writeTest = expectedValue: name: test:
writeDash "run-${name}" ''
if test "$(${test})" != "${expectedValue}"; then
echo 'test ${test} failed'
exit 1
fi
'';
data = {
json = expectDataEqual {
file = writeJSON "data.json" { hello = "world"; };
expected = ''
{
"hello": "world"
}
'';
};
in runCommand "test-writers" {
passthru = { inherit writeTest bin simple path; };
meta.platforms = lib.platforms.all;
} ''
${lib.concatMapStringsSep "\n" (test: writeTest "success" test.name "${test}/bin/${test.name}") (lib.attrValues bin)}
${lib.concatMapStringsSep "\n" (test: writeTest "success" test.name test) (lib.attrValues simple)}
${lib.concatMapStringsSep "\n" (test: writeTest "success" test.name test) (lib.attrValues path)}
echo 'nix-writers successfully tested' >&2
touch $out
''
toml = expectDataEqual {
file = writeTOML "data.toml" { hello = "world"; };
expected = "hello = 'world'\n";
};
yaml = expectDataEqual {
file = writeYAML "data.yaml" { hello = "world"; };
expected = "hello: world\n";
};
};
}

View file

@ -5,6 +5,5 @@ lib.recurseIntoAttrs {
cabalSdist = callPackage ./cabalSdist { };
documentationTarball = callPackage ./documentationTarball { };
setBuildTarget = callPackage ./setBuildTarget { };
writers = callPackage ./writers { };
incremental = callPackage ./incremental { };
}

View file

@ -1,26 +0,0 @@
# Wrap only the haskell-related tests from tests.writers
# in their own derivation for Hydra CI in the haskell-updates
# jobset. Can presumably removed as soon as tests.writers is
# always green on darwin as well:
# https://github.com/NixOS/nixpkgs/issues/126182
{ runCommand, tests }:
let
inherit (tests.writers)
writeTest
bin
simple
path
;
in
runCommand "test-haskell-writers" {
meta = {
inherit (tests.writers.meta) platforms;
};
} ''
${writeTest "success" "test-haskell-bin-writer" "${bin.haskell}/bin/${bin.haskell.name}"}
${writeTest "success" "test-haskell-simple-writer" simple.haskell}
${writeTest "success" "test-haskell-path-writer" path.haskell}
touch $out
''