linux: rewrite updater-script, make data-driven

Co-authored-by: Raito Bezarius <masterancpp@gmail.com>
This commit is contained in:
K900 2023-09-19 15:12:10 +03:00
parent ed5ab56d4c
commit 662a2c460d
14 changed files with 189 additions and 240 deletions

View file

@ -0,0 +1,38 @@
{
"testing": {
"version": "6.6-rc1",
"hash": "02zh3dnikyhhlas9xccia963d4yqmzq0m4b8s10x8mjng3na45hd"
},
"6.5": {
"version": "6.5.4",
"hash": "0s8nzd8yaq06bq8byk7aakbk95gh0rhlif26h1biw94v48anrxxx"
},
"6.4": {
"version": "6.4.16",
"hash": "0zgj1z97jyx7wf12zrnlcp0mj4cl43ais9qsy6dh1jwylf2fq9ln"
},
"6.1": {
"version": "6.1.54",
"hash": "09sfrq2l8f777mx2n9mhb6bgz1064bl04921byqnmk87si31w653"
},
"5.15": {
"version": "5.15.132",
"hash": "1b0qjsaqjw2rk86shmmrj2aasblkn27acjmc761vnjg7sv2baxs1"
},
"5.10": {
"version": "5.10.195",
"hash": "0n4vg2i9sq89wnz85arlyvwysh9s83cgzs5bk2wh98bivi5fwfs1"
},
"5.4": {
"version": "5.4.256",
"hash": "0fim5q9xakwnjfg48bpsic9r2r8dvrjlalqqkm9vh1rml9mhi967"
},
"4.19": {
"version": "4.19.294",
"hash": "03x0xsb8a369zdr81hg6xdl5n5v48k6iwnhj6r29725777lvvbfc"
},
"4.14": {
"version": "4.14.325",
"hash": "117p1mdha57f6d3kdwac9jrbmib7g77q4xhir8ghl6fmrs1f2sav"
}
}

View file

@ -1,18 +0,0 @@
{ lib, buildPackages, fetchurl, perl, buildLinux, nixosTests, ... } @ args:
with lib;
buildLinux (args // rec {
version = "4.14.325";
# modDirVersion needs to be x.y.z, will automatically add .0 if needed
modDirVersion = versions.pad 3 version;
# branchVersion needs to be x.y
extraMeta.branch = versions.majorMinor version;
src = fetchurl {
url = "mirror://kernel/linux/kernel/v4.x/linux-${version}.tar.xz";
sha256 = "117p1mdha57f6d3kdwac9jrbmib7g77q4xhir8ghl6fmrs1f2sav";
};
} // (args.argsOverride or {}))

View file

@ -1,18 +0,0 @@
{ lib, buildPackages, fetchurl, perl, buildLinux, nixosTests, ... } @ args:
with lib;
buildLinux (args // rec {
version = "4.19.294";
# modDirVersion needs to be x.y.z, will automatically add .0 if needed
modDirVersion = versions.pad 3 version;
# branchVersion needs to be x.y
extraMeta.branch = versions.majorMinor version;
src = fetchurl {
url = "mirror://kernel/linux/kernel/v4.x/linux-${version}.tar.xz";
sha256 = "03x0xsb8a369zdr81hg6xdl5n5v48k6iwnhj6r29725777lvvbfc";
};
} // (args.argsOverride or {}))

View file

@ -1,18 +0,0 @@
{ lib, buildPackages, fetchurl, perl, buildLinux, nixosTests, ... } @ args:
with lib;
buildLinux (args // rec {
version = "5.10.195";
# modDirVersion needs to be x.y.z, will automatically add .0 if needed
modDirVersion = versions.pad 3 version;
# branchVersion needs to be x.y
extraMeta.branch = versions.majorMinor version;
src = fetchurl {
url = "mirror://kernel/linux/kernel/v5.x/linux-${version}.tar.xz";
sha256 = "0n4vg2i9sq89wnz85arlyvwysh9s83cgzs5bk2wh98bivi5fwfs1";
};
} // (args.argsOverride or {}))

View file

@ -1,18 +0,0 @@
{ lib, buildPackages, fetchurl, perl, buildLinux, nixosTests, ... } @ args:
with lib;
buildLinux (args // rec {
version = "5.15.132";
# modDirVersion needs to be x.y.z, will automatically add .0 if needed
modDirVersion = versions.pad 3 version;
# branchVersion needs to be x.y
extraMeta.branch = versions.majorMinor version;
src = fetchurl {
url = "mirror://kernel/linux/kernel/v5.x/linux-${version}.tar.xz";
sha256 = "1b0qjsaqjw2rk86shmmrj2aasblkn27acjmc761vnjg7sv2baxs1";
};
} // (args.argsOverride or { }))

View file

@ -1,18 +0,0 @@
{ lib, buildPackages, fetchurl, perl, buildLinux, nixosTests, ... } @ args:
with lib;
buildLinux (args // rec {
version = "5.4.256";
# modDirVersion needs to be x.y.z, will automatically add .0 if needed
modDirVersion = versions.pad 3 version;
# branchVersion needs to be x.y
extraMeta.branch = versions.majorMinor version;
src = fetchurl {
url = "mirror://kernel/linux/kernel/v5.x/linux-${version}.tar.xz";
sha256 = "0fim5q9xakwnjfg48bpsic9r2r8dvrjlalqqkm9vh1rml9mhi967";
};
} // (args.argsOverride or {}))

View file

@ -1,18 +0,0 @@
{ lib, buildPackages, fetchurl, perl, buildLinux, nixosTests, ... } @ args:
with lib;
buildLinux (args // rec {
version = "6.1.54";
# modDirVersion needs to be x.y.z, will automatically add .0 if needed
modDirVersion = versions.pad 3 version;
# branchVersion needs to be x.y
extraMeta.branch = versions.majorMinor version;
src = fetchurl {
url = "mirror://kernel/linux/kernel/v6.x/linux-${version}.tar.xz";
sha256 = "09sfrq2l8f777mx2n9mhb6bgz1064bl04921byqnmk87si31w653";
};
} // (args.argsOverride or { }))

View file

@ -1,18 +0,0 @@
{ lib, fetchurl, buildLinux, ... } @ args:
with lib;
buildLinux (args // rec {
version = "6.4.16";
# modDirVersion needs to be x.y.z, will automatically add .0 if needed
modDirVersion = versions.pad 3 version;
# branchVersion needs to be x.y
extraMeta.branch = versions.majorMinor version;
src = fetchurl {
url = "mirror://kernel/linux/kernel/v6.x/linux-${version}.tar.xz";
sha256 = "0zgj1z97jyx7wf12zrnlcp0mj4cl43ais9qsy6dh1jwylf2fq9ln";
};
} // (args.argsOverride or { }))

View file

@ -1,18 +0,0 @@
{ lib, fetchurl, buildLinux, ... } @ args:
with lib;
buildLinux (args // rec {
version = "6.5.4";
# modDirVersion needs to be x.y.z, will automatically add .0 if needed
modDirVersion = versions.pad 3 version;
# branchVersion needs to be x.y
extraMeta.branch = versions.majorMinor version;
src = fetchurl {
url = "mirror://kernel/linux/kernel/v6.x/linux-${version}.tar.xz";
sha256 = "0s8nzd8yaq06bq8byk7aakbk95gh0rhlif26h1biw94v48anrxxx";
};
} // (args.argsOverride or { }))

View file

@ -1,20 +0,0 @@
{ lib, buildPackages, fetchzip, perl, buildLinux, nixosTests, ... } @ args:
with lib;
buildLinux (args // rec {
version = "6.6-rc1";
extraMeta.branch = lib.versions.majorMinor version;
# modDirVersion needs to be x.y.z, will always add .0
modDirVersion = versions.pad 3 version;
src = fetchzip {
url = "https://git.kernel.org/torvalds/t/linux-${version}.tar.gz";
hash = "sha256-DRai7HhWVtRB0GiRCvCv2JM2TFKRsZ60ohD6GW0b8As=";
};
# Should the testing kernels ever be built on Hydra?
extraMeta.hydraPlatforms = [];
} // (args.argsOverride or {}))

View file

@ -0,0 +1,18 @@
{ branch, lib, fetchurl, buildLinux, ... } @ args:
let
allKernels = builtins.fromJSON (builtins.readFile ./kernels-org.json);
thisKernel = allKernels.${branch};
args' = (builtins.removeAttrs args ["branch"]) // rec {
inherit (thisKernel) version;
modDirVersion = lib.versions.pad 3 version;
extraMeta.branch = branch;
src = fetchurl {
url = "mirror://kernel/linux/kernel/v${lib.versions.major version}.x/linux-${version}.tar.xz";
sha256 = thisKernel.hash;
};
} // (args.argsOverride or {});
in
buildLinux args'

View file

@ -0,0 +1,104 @@
#!/usr/bin/env nix-shell
#!nix-shell -i python3 -p "python3.withPackages (ps: [ ps.beautifulsoup4 ps.lxml ])"
from enum import Enum
from bs4 import BeautifulSoup, NavigableString, Tag
from dataclasses import dataclass
import json
import pathlib
import re
import subprocess
import urllib.request
import sys
HERE = pathlib.Path(__file__).parent
ROOT = HERE.parent.parent.parent.parent
VERSIONS_FILE = HERE / "kernels-org.json"
class KernelNature(Enum):
MAINLINE = 1
STABLE = 2
LONGTERM = 3
@dataclass
class KernelRelease:
nature: KernelNature
version: str
date: str
link: str
eol: bool = False
def parse_release(release: Tag) -> KernelRelease | None:
columns: list[Tag] = list(release.find_all('td'))
try:
nature = KernelNature[columns[0].get_text().rstrip(':').upper()]
except KeyError:
return None
version = columns[1].get_text().rstrip(' [EOL]')
date = columns[2].get_text()
link = columns[3].find('a')
if link is not None and isinstance(link, Tag):
link = link.attrs.get('href')
assert link is not None, f'link for kernel {version} is non-existent'
eol = bool(release.find(class_='eolkernel'))
return KernelRelease(nature=nature, version=version, date=date, link=link, eol=eol)
def get_branch(version: str):
# This is a testing kernel.
if 'rc' in version:
return 'testing'
else:
major, minor, *_ = version.split(".")
return f"{major}.{minor}"
def get_hash(url: str):
return subprocess.check_output(["nix-prefetch-url", url]).decode().strip()
def commit(message):
return subprocess.check_call(["git", "commit", "-m", message, VERSIONS_FILE])
def main():
kernel_org = urllib.request.urlopen("https://kernel.org/")
soup = BeautifulSoup(kernel_org.read().decode(), "lxml")
release_table = soup.find(id='releases')
if not release_table or isinstance(release_table, NavigableString):
print(release_table)
print('Failed to find the release table on https://kernel.org')
sys.exit(1)
releases = release_table.find_all('tr')
parsed_releases = filter(None, [parse_release(release) for release in releases])
all_kernels = json.load(VERSIONS_FILE.open())
for kernel in parsed_releases:
branch = get_branch(kernel.version)
nixpkgs_branch = branch.replace('.', '_')
old_version = all_kernels.get(branch, {}).get("version")
if old_version == kernel.version:
print(f"linux_{nixpkgs_branch}: {kernel.version} is latest, skipping...")
continue
if old_version is None:
message = f"linux_{nixpkgs_branch}: init at {kernel.version}"
else:
message = f"linux_{nixpkgs_branch}: {old_version} -> {kernel.version}"
print(message)
all_kernels[branch] = {"version": kernel.version, "hash": get_hash(kernel.link)}
with VERSIONS_FILE.open("w") as fd:
json.dump(all_kernels, fd, indent=4)
fd.write("\n") # makes editorconfig happy
commit(message)
if __name__ == "__main__":
main()

View file

@ -1,72 +1,14 @@
#!/usr/bin/env bash
set -e
cd "$(dirname "$(readlink -f "$0")")" || exit
# Get the latest versions from kernel.org
LINUXSED='s/.*linux-\([0-9]\+\(.[0-9]\+\)*\).*/\1/p'
KDATA="$(curl -s https://www.kernel.org | sed -n -e '/Download complete/p')"
VERSIONS=($(sed -n -e $LINUXSED <<< "$KDATA" | sort -Vr))
echo "Update linux (mainline)"
COMMIT=1 ./update-mainline.py || echo "update-mainline failed with exit code $?"
# Remove mainline version if there is a stable update
# Note due to sorting these two will always exist at the bottom
if grep -q "^${VERSIONS[1]}" <<< "${VERSIONS[0]}"; then
VERSIONS=(${VERSIONS[@]:0:1} ${VERSIONS[@]:2})
fi
echo "Update linux-rt"
COMMIT=1 ./update-rt.sh || echo "update-rt failed with exit code $?"
# Inspect each file and see if it has the latest version
NIXPKGS="$(git rev-parse --show-toplevel)"
ls $NIXPKGS/pkgs/os-specific/linux/kernel | while read FILE; do
KERNEL="$(sed -n -e $LINUXSED <<< "$FILE")"
[ -z "$KERNEL" ] && continue
echo "Update linux-libre"
COMMIT=1 ./update-libre.sh || echo "update-libre failed with exit code $?"
# Find the matching new kernel version
MATCHING=""
for V in "${VERSIONS[@]}"; do
if grep -q "^$KERNEL" <<< "$V"; then
MATCHING="$V"
break
fi
done
if [ -z "$MATCHING" ]; then
echo "Out-of-support $KERNEL"
continue
fi
# Inspect the nix expression to check for changes
DATA="$(<$NIXPKGS/pkgs/os-specific/linux/kernel/$FILE)"
URL="$(sed -n -e 's/.*url = "\(.*\)";.*/\1/p' <<< "$DATA" | sed -e "s/\${version}/$MATCHING/g")"
OLDVER=$(sed -n -e 's/.*version = "\(.*\)".*/\1/p' <<< "$DATA")
if [ "$OLDVER" = "$V" ]; then
echo "No updates for $KERNEL"
continue
fi
# Download the new file for the hash
if ! HASH="$(nix-prefetch-url $URL 2>/dev/null)"; then
echo "Failed to get hash of $URL"
continue
fi
sed -i -e "s/sha256 = \".*\"/sha256 = \"$HASH\"/g" $NIXPKGS/pkgs/os-specific/linux/kernel/$FILE
# Rewrite the expression
sed -i -e '/version = /d' $NIXPKGS/pkgs/os-specific/linux/kernel/$FILE
sed -i -e "\#buildLinux (args // rec {#a \ version = \"$V\";" $NIXPKGS/pkgs/os-specific/linux/kernel/$FILE
# Commit the changes
git add -u $NIXPKGS/pkgs/os-specific/linux/kernel/$FILE
git commit -m "linux: $OLDVER -> $V" >/dev/null 2>&1
echo "Updated $OLDVER -> $V"
done
# Allowing errors again: one broken update script shouldn't inhibit the
# update of other kernel variants.
set +e
echo Update linux-rt
COMMIT=1 $NIXPKGS/pkgs/os-specific/linux/kernel/update-rt.sh || echo "update-rt failed with exit code $?"
echo Update linux-libre
COMMIT=1 $NIXPKGS/pkgs/os-specific/linux/kernel/update-libre.sh || echo "update-libre failed with exit code $?"
echo Update linux-hardened
COMMIT=1 $NIXPKGS/pkgs/os-specific/linux/kernel/hardened/update.py || echo "update-hardened failed with exit code $?"
echo "Update linux-hardened"
COMMIT=1 ./hardened/update.py || echo "update-hardened failed with exit code $?"

View file

@ -96,7 +96,8 @@ in {
rpiVersion = 4;
};
linux_4_14 = callPackage ../os-specific/linux/kernel/linux-4.14.nix {
linux_4_14 = callPackage ../os-specific/linux/kernel/mainline.nix {
branch = "4.14";
kernelPatches =
[ kernelPatches.bridge_stp_helper
kernelPatches.request_key_helper
@ -107,7 +108,8 @@ in {
];
};
linux_4_19 = callPackage ../os-specific/linux/kernel/linux-4.19.nix {
linux_4_19 = callPackage ../os-specific/linux/kernel/mainline.nix {
branch = "4.19";
kernelPatches =
[ kernelPatches.bridge_stp_helper
kernelPatches.request_key_helper
@ -115,7 +117,8 @@ in {
];
};
linux_5_4 = callPackage ../os-specific/linux/kernel/linux-5.4.nix {
linux_5_4 = callPackage ../os-specific/linux/kernel/mainline.nix {
branch = "5.4";
kernelPatches = [
kernelPatches.bridge_stp_helper
kernelPatches.request_key_helper
@ -130,7 +133,8 @@ in {
];
};
linux_5_10 = callPackage ../os-specific/linux/kernel/linux-5.10.nix {
linux_5_10 = callPackage ../os-specific/linux/kernel/mainline.nix {
branch = "5.10";
kernelPatches = [
kernelPatches.bridge_stp_helper
kernelPatches.request_key_helper
@ -145,7 +149,8 @@ in {
];
};
linux_5_15 = callPackage ../os-specific/linux/kernel/linux-5.15.nix {
linux_5_15 = callPackage ../os-specific/linux/kernel/mainline.nix {
branch = "5.15";
kernelPatches = [
kernelPatches.bridge_stp_helper
kernelPatches.request_key_helper
@ -160,7 +165,8 @@ in {
];
};
linux_6_1 = callPackage ../os-specific/linux/kernel/linux-6.1.nix {
linux_6_1 = callPackage ../os-specific/linux/kernel/mainline.nix {
branch = "6.1";
kernelPatches = [
kernelPatches.bridge_stp_helper
kernelPatches.request_key_helper
@ -177,7 +183,8 @@ in {
];
};
linux_6_4 = callPackage ../os-specific/linux/kernel/linux-6.4.nix {
linux_6_4 = callPackage ../os-specific/linux/kernel/mainline.nix {
branch = "6.4";
kernelPatches = [
kernelPatches.bridge_stp_helper
kernelPatches.request_key_helper
@ -185,7 +192,8 @@ in {
];
};
linux_6_5 = callPackage ../os-specific/linux/kernel/linux-6.5.nix {
linux_6_5 = callPackage ../os-specific/linux/kernel/mainline.nix {
branch = "6.5";
kernelPatches = [
kernelPatches.bridge_stp_helper
kernelPatches.request_key_helper
@ -194,7 +202,10 @@ in {
};
linux_testing = let
testing = callPackage ../os-specific/linux/kernel/linux-testing.nix {
testing = callPackage ../os-specific/linux/kernel/mainline.nix {
# A special branch that tracks the kernel under the release process
# i.e. which has at least a public rc1 and is not released yet.
branch = "testing";
kernelPatches = [
kernelPatches.bridge_stp_helper
kernelPatches.request_key_helper