Add benchmarking scripts
These scripts were originally written by horrors, and have since been hacked up a lot by jade. We are putting them up as a CL since it is better to have checked in benchmarking scripts than to not have benchmarking scripts. cc: https://git.lix.systems/lix-project/lix/issues/23 Co-authored-by: eldritch horrors <pennae@lix.systems> Change-Id: I95c2f9d24753ac468944c5781deec9508fd5cb8c
This commit is contained in:
Jade Lovelace
parent
1e74bffd5c
commit
f4fc2b5d10
5 changed files with 503 additions and 0 deletions
3
bench/.gitignore
vendored
Normal file
3
bench/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
bench-*.json
|
||||||
|
bench-*.md
|
||||||
|
nixpkgs
|
||||||
91
bench/README.md
Normal file
91
bench/README.md
Normal file
|
|
@ -0,0 +1,91 @@
|
||||||
|
# Benchmarking scripts for Lix
|
||||||
|
|
||||||
|
These are very much WIP, and have a few clumsy assumptions that we would
|
||||||
|
somewhat rather be fixed, but we have committed them to let others be able to
|
||||||
|
do benchmarking in the mean time.
|
||||||
|
|
||||||
|
## Benchmarking procedure
|
||||||
|
|
||||||
|
Build some Lixes you want to compare, by whichever means you wish.
|
||||||
|
|
||||||
|
Get a computer that is not busy and *strongly preferably* is bare-metal or at
|
||||||
|
least not a cloud VM (e.g. go make coffee when running benchmarks).
|
||||||
|
|
||||||
|
From the root of a Lix checkout, run `./bench/bench.sh resultlink-one
|
||||||
|
resultlink-two`, where `resultlink-one` and `resultlink-two` are the result
|
||||||
|
links from the builds you want to test (they can be any directory with bin/nix
|
||||||
|
in it, however).
|
||||||
|
|
||||||
|
To get the summary again, run `./bench/summarize.jq bench/bench-*.json`.
|
||||||
|
|
||||||
|
## Example results
|
||||||
|
|
||||||
|
(vim tip: `:r !bench/summarize.jq bench/bench-*.json` to dump it directly into
|
||||||
|
your editor)
|
||||||
|
|
||||||
|
```
|
||||||
|
result-asserts/bin/nix --extra-experimental-features 'nix-command flakes' search --no-eval-cache github:nixos/nixpkgs/e1fa12d4f6
|
||||||
|
c6fe19ccb59cac54b5b3f25e160870 hello
|
||||||
|
mean: 15.993s ± 0.081s
|
||||||
|
user: 13.321s | system: 1.865s
|
||||||
|
median: 15.994s
|
||||||
|
range: 15.829s ... 16.096s
|
||||||
|
relative: 1
|
||||||
|
result/bin/nix --extra-experimental-features 'nix-command flakes' search --no-eval-cache github:nixos/nixpkgs/e1fa12d4f6c6fe19cc
|
||||||
|
b59cac54b5b3f25e160870 hello
|
||||||
|
mean: 15.897s ± 0.075s
|
||||||
|
user: 13.248s | system: 1.843s
|
||||||
|
median: 15.88s
|
||||||
|
range: 15.807s ... 16.047s
|
||||||
|
relative: 0.994
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
result/bin/nix --extra-experimental-features 'nix-command flakes' eval -f bench/nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix
|
||||||
|
mean: 0.4s ± 0.024s
|
||||||
|
user: 0.335s | system: 0.046s
|
||||||
|
median: 0.386s
|
||||||
|
range: 0.379s ... 0.43s
|
||||||
|
relative: 1
|
||||||
|
|
||||||
|
result-asserts/bin/nix --extra-experimental-features 'nix-command flakes' eval -f bench/nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix
|
||||||
|
mean: 0.404s ± 0.024s
|
||||||
|
user: 0.338s | system: 0.046s
|
||||||
|
median: 0.386s
|
||||||
|
range: 0.384s ... 0.436s
|
||||||
|
relative: 1.008
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
result-asserts/bin/nix --extra-experimental-features 'nix-command flakes' eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'
|
||||||
|
mean: 5.838s ± 0.023s
|
||||||
|
user: 5.083s | system: 0.464s
|
||||||
|
median: 5.845s
|
||||||
|
range: 5.799s ... 5.867s
|
||||||
|
relative: 1
|
||||||
|
|
||||||
|
result/bin/nix --extra-experimental-features 'nix-command flakes' eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'
|
||||||
|
mean: 5.788s ± 0.044s
|
||||||
|
user: 5.056s | system: 0.439s
|
||||||
|
median: 5.79s
|
||||||
|
range: 5.715s ... 5.876s
|
||||||
|
relative: 0.991
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
GC_INITIAL_HEAP_SIZE=10g result-asserts/bin/nix eval --extra-experimental-features 'nix-command flakes' --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'
|
||||||
|
mean: 4.147s ± 0.021s
|
||||||
|
user: 3.457s | system: 0.487s
|
||||||
|
median: 4.147s
|
||||||
|
range: 4.123s ... 4.195s
|
||||||
|
relative: 1
|
||||||
|
|
||||||
|
GC_INITIAL_HEAP_SIZE=10g result/bin/nix eval --extra-experimental-features 'nix-command flakes' --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'
|
||||||
|
mean: 4.149s ± 0.027s
|
||||||
|
user: 3.483s | system: 0.456s
|
||||||
|
median: 4.142s
|
||||||
|
range: 4.126s ... 4.215s
|
||||||
|
relative: 1
|
||||||
|
|
||||||
|
---
|
||||||
|
```
|
||||||
62
bench/bench.sh
Executable file
62
bench/bench.sh
Executable file
|
|
@ -0,0 +1,62 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
shopt -s inherit_errexit
|
||||||
|
|
||||||
|
scriptdir=$(cd "$(dirname -- "$0")" ; pwd -P)
|
||||||
|
cd "$scriptdir/.."
|
||||||
|
|
||||||
|
if [[ $# -lt 2 ]]; then
|
||||||
|
# FIXME(jade): it is a reasonable use case to want to run a benchmark run
|
||||||
|
# on just one build. However, since we are using hyperfine in comparison
|
||||||
|
# mode, we would have to combine the JSON ourselves to support that, which
|
||||||
|
# would probably be better done by writing a benchmarking script in
|
||||||
|
# not-bash.
|
||||||
|
echo "Fewer than two result dirs given, nothing to compare!" >&2
|
||||||
|
echo "Pass some directories (with names indicating which alternative they are) with bin/nix in them" >&2
|
||||||
|
echo "Usage: ./bench/bench.sh result-1 result-2 [result-3...]" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
_exit=""
|
||||||
|
trap "$_exit" EXIT
|
||||||
|
|
||||||
|
# XXX: yes this is very silly. flakes~!!
|
||||||
|
nix build --impure --expr '(builtins.getFlake "git+file:.").inputs.nixpkgs.outPath' -o bench/nixpkgs
|
||||||
|
|
||||||
|
export NIX_REMOTE="$(mktemp -d)"
|
||||||
|
_exit='rm -rfv "$NIX_REMOTE"; $_exit'
|
||||||
|
export NIX_PATH="nixpkgs=bench/nixpkgs:nixos-config=bench/configuration.nix"
|
||||||
|
|
||||||
|
builds=("$@")
|
||||||
|
|
||||||
|
flake_args="--extra-experimental-features 'nix-command flakes'"
|
||||||
|
|
||||||
|
hyperfineArgs=(
|
||||||
|
--parameter-list BUILD "$(IFS=,; echo "${builds[*]}")"
|
||||||
|
--warmup 2 --runs 10
|
||||||
|
)
|
||||||
|
|
||||||
|
declare -A cases
|
||||||
|
cases=(
|
||||||
|
[search]="{BUILD}/bin/nix $flake_args search --no-eval-cache github:nixos/nixpkgs/e1fa12d4f6c6fe19ccb59cac54b5b3f25e160870 hello"
|
||||||
|
[rebuild]="{BUILD}/bin/nix $flake_args eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'"
|
||||||
|
[rebuild-lh]="GC_INITIAL_HEAP_SIZE=10g {BUILD}/bin/nix eval $flake_args --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'"
|
||||||
|
[parse]="{BUILD}/bin/nix $flake_args eval -f bench/nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix"
|
||||||
|
)
|
||||||
|
|
||||||
|
benches=(
|
||||||
|
rebuild
|
||||||
|
rebuild-lh
|
||||||
|
search
|
||||||
|
parse
|
||||||
|
)
|
||||||
|
|
||||||
|
for k in "${benches[@]}"; do
|
||||||
|
taskset -c 2,3 \
|
||||||
|
chrt -f 50 \
|
||||||
|
hyperfine "${hyperfineArgs[@]}" --export-json="bench/bench-${k}.json" --export-markdown="bench/bench-${k}.md" "${cases[$k]}"
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "Benchmarks summary (from ./bench/summarize.jq bench/bench-*.json)"
|
||||||
|
bench/summarize.jq bench/*.json
|
||||||
325
bench/configuration.nix
Normal file
325
bench/configuration.nix
Normal file
|
|
@ -0,0 +1,325 @@
|
||||||
|
{
|
||||||
|
config,
|
||||||
|
pkgs,
|
||||||
|
lib,
|
||||||
|
...
|
||||||
|
}:
|
||||||
|
|
||||||
|
{
|
||||||
|
boot = {
|
||||||
|
initrd = {
|
||||||
|
availableKernelModules = [
|
||||||
|
"xhci_pci"
|
||||||
|
"ahci"
|
||||||
|
];
|
||||||
|
kernelModules = [ "dm-snapshot" ];
|
||||||
|
luks.devices = {
|
||||||
|
croot = {
|
||||||
|
device = "/dev/sdb";
|
||||||
|
allowDiscards = true;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
kernelModules = [ "kvm-intel" ];
|
||||||
|
kernelPackages = pkgs.linuxPackages_latest;
|
||||||
|
|
||||||
|
loader = {
|
||||||
|
systemd-boot.enable = true;
|
||||||
|
efi.canTouchEfiVariables = true;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
hardware = {
|
||||||
|
enableRedistributableFirmware = true;
|
||||||
|
cpu.intel.updateMicrocode = true;
|
||||||
|
opengl.driSupport32Bit = true;
|
||||||
|
opengl.extraPackages = with pkgs; [
|
||||||
|
vaapiIntel
|
||||||
|
intel-media-driver
|
||||||
|
intel-compute-runtime
|
||||||
|
];
|
||||||
|
};
|
||||||
|
|
||||||
|
fileSystems = {
|
||||||
|
"/" = {
|
||||||
|
device = "/dev/sda2";
|
||||||
|
fsType = "xfs";
|
||||||
|
options = [ "noatime" ];
|
||||||
|
};
|
||||||
|
|
||||||
|
"/boot" = {
|
||||||
|
device = "/dev/sda1";
|
||||||
|
fsType = "vfat";
|
||||||
|
};
|
||||||
|
|
||||||
|
"/nas" = {
|
||||||
|
device = "nas:/";
|
||||||
|
fsType = "nfs4";
|
||||||
|
options = [
|
||||||
|
"ro"
|
||||||
|
"x-systemd.automount"
|
||||||
|
];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
swapDevices = [ { device = "/dev/swap"; } ];
|
||||||
|
|
||||||
|
networking = {
|
||||||
|
useDHCP = false;
|
||||||
|
hostName = "host";
|
||||||
|
wireless = {
|
||||||
|
enable = true;
|
||||||
|
interfaces = [ "eth1" ];
|
||||||
|
};
|
||||||
|
interfaces = {
|
||||||
|
eth0.useDHCP = true;
|
||||||
|
eth1.useDHCP = true;
|
||||||
|
};
|
||||||
|
wg-quick.interfaces = {
|
||||||
|
wg0 = {
|
||||||
|
address = [ "2001:db8::1" ];
|
||||||
|
privateKeyFile = "/etc/secrets/wg0.key";
|
||||||
|
peers = [
|
||||||
|
{
|
||||||
|
publicKey = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";
|
||||||
|
endpoint = "[2001:db8::2]:61021";
|
||||||
|
allowedIPs = [ "2001::db8:1::/64" ];
|
||||||
|
}
|
||||||
|
];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
firewall.allowedUDPPorts = [ 4567 ];
|
||||||
|
};
|
||||||
|
|
||||||
|
i18n = {
|
||||||
|
defaultLocale = "en_US.UTF-8";
|
||||||
|
inputMethod.enabled = "ibus";
|
||||||
|
};
|
||||||
|
|
||||||
|
services = {
|
||||||
|
xserver = {
|
||||||
|
enable = true;
|
||||||
|
layout = "us";
|
||||||
|
xkbVariant = "altgr-intl";
|
||||||
|
xkbOptions = "ctrl:nocaps";
|
||||||
|
libinput.enable = true;
|
||||||
|
wacom.enable = true;
|
||||||
|
videoDrivers = [ "modesetting" ];
|
||||||
|
modules = [ pkgs.xf86_input_wacom ];
|
||||||
|
|
||||||
|
displayManager.sx.enable = true;
|
||||||
|
windowManager.i3.enable = true;
|
||||||
|
};
|
||||||
|
|
||||||
|
udev.extraHwdb = ''
|
||||||
|
# not like this mattered at all
|
||||||
|
# we're not running udev from here
|
||||||
|
'';
|
||||||
|
|
||||||
|
udev.extraRules = ''
|
||||||
|
# ACTION=="add", SUBSYSTEM=="input", ...
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
sound.enable = true;
|
||||||
|
hardware.pulseaudio = {
|
||||||
|
enable = true;
|
||||||
|
package = pkgs.pulseaudioFull;
|
||||||
|
daemon.config = {
|
||||||
|
lock-memory = "yes";
|
||||||
|
realtime-scheduling = "yes";
|
||||||
|
rlimit-rtprio = "-1";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
programs = {
|
||||||
|
light.enable = true;
|
||||||
|
wireshark = {
|
||||||
|
enable = true;
|
||||||
|
package = pkgs.wireshark-qt;
|
||||||
|
};
|
||||||
|
gnupg.agent = {
|
||||||
|
enable = true;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
fonts.packages = with pkgs; [
|
||||||
|
font-awesome
|
||||||
|
noto-fonts
|
||||||
|
noto-fonts-cjk
|
||||||
|
noto-fonts-emoji
|
||||||
|
noto-fonts-extra
|
||||||
|
dejavu_fonts
|
||||||
|
powerline-fonts
|
||||||
|
source-code-pro
|
||||||
|
cantarell-fonts
|
||||||
|
];
|
||||||
|
|
||||||
|
users = {
|
||||||
|
mutableUsers = false;
|
||||||
|
|
||||||
|
users = {
|
||||||
|
user = {
|
||||||
|
isNormalUser = true;
|
||||||
|
group = "user";
|
||||||
|
extraGroups = [
|
||||||
|
"wheel"
|
||||||
|
"video"
|
||||||
|
"audio"
|
||||||
|
"dialout"
|
||||||
|
"users"
|
||||||
|
"kvm"
|
||||||
|
"wireshark"
|
||||||
|
];
|
||||||
|
password = "unimportant";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
groups = {
|
||||||
|
user = { };
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
security = {
|
||||||
|
pam.loginLimits = [
|
||||||
|
{
|
||||||
|
domain = "@audio";
|
||||||
|
item = "memlock";
|
||||||
|
type = "-";
|
||||||
|
value = "unlimited";
|
||||||
|
}
|
||||||
|
{
|
||||||
|
domain = "@audio";
|
||||||
|
item = "rtprio";
|
||||||
|
type = "-";
|
||||||
|
value = "99";
|
||||||
|
}
|
||||||
|
{
|
||||||
|
domain = "@audio";
|
||||||
|
item = "nofile";
|
||||||
|
type = "soft";
|
||||||
|
value = "99999";
|
||||||
|
}
|
||||||
|
{
|
||||||
|
domain = "@audio";
|
||||||
|
item = "nofile";
|
||||||
|
type = "hard";
|
||||||
|
value = "99999";
|
||||||
|
}
|
||||||
|
];
|
||||||
|
|
||||||
|
sudo.extraRules = [
|
||||||
|
{
|
||||||
|
users = [ "user" ];
|
||||||
|
commands = [
|
||||||
|
{
|
||||||
|
command = "${pkgs.linuxPackages.cpupower}/bin/cpupower";
|
||||||
|
options = [ "NOPASSWD" ];
|
||||||
|
}
|
||||||
|
];
|
||||||
|
}
|
||||||
|
];
|
||||||
|
};
|
||||||
|
|
||||||
|
environment.systemPackages = with pkgs; [
|
||||||
|
a2jmidid
|
||||||
|
age
|
||||||
|
ardour
|
||||||
|
bemenu
|
||||||
|
blender
|
||||||
|
breeze-icons
|
||||||
|
breeze-qt5
|
||||||
|
bubblewrap
|
||||||
|
calf
|
||||||
|
claws-mail
|
||||||
|
darktable
|
||||||
|
duperemove
|
||||||
|
emacs
|
||||||
|
feh
|
||||||
|
file
|
||||||
|
firefox
|
||||||
|
fluidsynth
|
||||||
|
gnome3.adwaita-icon-theme
|
||||||
|
gnuplot
|
||||||
|
graphviz
|
||||||
|
helm
|
||||||
|
i3status-rust
|
||||||
|
inkscape
|
||||||
|
jack2
|
||||||
|
jq
|
||||||
|
krita
|
||||||
|
ldns
|
||||||
|
libqalculate
|
||||||
|
libreoffice
|
||||||
|
man-pages
|
||||||
|
nheko
|
||||||
|
nix-diff
|
||||||
|
nix-index
|
||||||
|
nix-output-monitor
|
||||||
|
open-music-kontrollers.patchmatrix
|
||||||
|
pamixer
|
||||||
|
pavucontrol
|
||||||
|
pciutils
|
||||||
|
picom
|
||||||
|
pwgen
|
||||||
|
redshift
|
||||||
|
ripgrep
|
||||||
|
rlwrap
|
||||||
|
silver-searcher
|
||||||
|
soundfont-fluid
|
||||||
|
whois
|
||||||
|
wol
|
||||||
|
xclip
|
||||||
|
xdot
|
||||||
|
xdotool
|
||||||
|
xorg.xkbcomp
|
||||||
|
yt-dlp
|
||||||
|
zathura
|
||||||
|
borgbackup
|
||||||
|
linuxPackages.cpupower
|
||||||
|
mtr
|
||||||
|
kitty
|
||||||
|
xf86_input_wacom
|
||||||
|
];
|
||||||
|
|
||||||
|
environment.pathsToLink = [ "/share/soundfonts" ];
|
||||||
|
|
||||||
|
systemd.user.services.run-python = {
|
||||||
|
after = [ "network-online.target" ];
|
||||||
|
script = ''
|
||||||
|
exec ${pkgs.python3}/bin/python
|
||||||
|
'';
|
||||||
|
serviceConfig = {
|
||||||
|
CapabilityBoundingSet = [ "" ];
|
||||||
|
KeyringMode = "private";
|
||||||
|
LockPersonality = true;
|
||||||
|
MemoryDenyWriteExecute = true;
|
||||||
|
NoNewPrivileges = true;
|
||||||
|
PrivateDevices = true;
|
||||||
|
PrivateTmp = true;
|
||||||
|
PrivateUsers = true;
|
||||||
|
ProcSubset = "pid";
|
||||||
|
ProtectClock = true;
|
||||||
|
ProtectControlGroups = true;
|
||||||
|
ProtectHome = true;
|
||||||
|
ProtectHostname = true;
|
||||||
|
ProtectKernelLogs = true;
|
||||||
|
ProtectKernelModules = true;
|
||||||
|
ProtectKernelTunables = true;
|
||||||
|
ProtectProc = "invisible";
|
||||||
|
ProtectSystem = "strict";
|
||||||
|
RestrictAddressFamilies = "AF_INET AF_INET6";
|
||||||
|
RestrictNamespaces = true;
|
||||||
|
RestrictRealtime = true;
|
||||||
|
RestrictSUIDSGID = true;
|
||||||
|
SystemCallArchitectures = "native";
|
||||||
|
SystemCallFilter = [
|
||||||
|
"@system-service"
|
||||||
|
"~ @resources @privileged"
|
||||||
|
];
|
||||||
|
UMask = "077";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
system.stateVersion = "23.11";
|
||||||
|
}
|
||||||
22
bench/summarize.jq
Executable file
22
bench/summarize.jq
Executable file
|
|
@ -0,0 +1,22 @@
|
||||||
|
#!/usr/bin/env -S jq -Mrf
|
||||||
|
|
||||||
|
def round3:
|
||||||
|
. * 1000 | round | . / 1000
|
||||||
|
;
|
||||||
|
|
||||||
|
def stats($first):
|
||||||
|
[
|
||||||
|
" mean: \(.mean | round3)s ± \(.stddev | round3)s",
|
||||||
|
" user: \(.user | round3)s | system: \(.system | round3)s",
|
||||||
|
" median: \(.median | round3)s",
|
||||||
|
" range: \(.min | round3)s ... \(.max | round3)s",
|
||||||
|
" relative: \(.mean / $first.mean | round3)"
|
||||||
|
]
|
||||||
|
| join("\n")
|
||||||
|
;
|
||||||
|
|
||||||
|
def fmt($first):
|
||||||
|
"\(.command)\n" + (. | stats($first))
|
||||||
|
;
|
||||||
|
|
||||||
|
[.results | .[0] as $first | .[] | fmt($first)] | join("\n\n") | (. + "\n\n---\n")
|
||||||
Loading…
Reference in a new issue