From 9121853fe555c56d11b31c8e3e8215dab8a49628 Mon Sep 17 00:00:00 2001 From: Sebastian Neubauer Date: Sat, 26 Aug 2023 18:56:30 +0200 Subject: [PATCH] vulkan-cts: Add update script --- pkgs/tools/graphics/vulkan-cts/default.nix | 67 ++----------- pkgs/tools/graphics/vulkan-cts/sources.nix | 73 +++++++++++++++ pkgs/tools/graphics/vulkan-cts/update.sh | 21 +++++ .../graphics/vulkan-cts/vk-cts-sources.py | 93 +++++++++++++++++++ 4 files changed, 194 insertions(+), 60 deletions(-) create mode 100644 pkgs/tools/graphics/vulkan-cts/sources.nix create mode 100755 pkgs/tools/graphics/vulkan-cts/update.sh create mode 100755 pkgs/tools/graphics/vulkan-cts/vk-cts-sources.py diff --git a/pkgs/tools/graphics/vulkan-cts/default.nix b/pkgs/tools/graphics/vulkan-cts/default.nix index bdbf573a016c..449d4a8c88de 100644 --- a/pkgs/tools/graphics/vulkan-cts/default.nix +++ b/pkgs/tools/graphics/vulkan-cts/default.nix @@ -32,54 +32,8 @@ let # It also expects the version specified in the repository, which can be incompatible # with the version in nixpkgs (e.g. for SPIRV-Headers), so we don't want to patch in our packages. # The revisions are extracted from https://github.com/KhronosGroup/VK-GL-CTS/blob/main/external/fetch_sources.py#L290 - amber = fetchFromGitHub { - owner = "google"; - repo = "amber"; - rev = "933ecb4d6288675a92eb1650e0f52b1d7afe8273"; - hash = "sha256-v9z4gv/mTjaCkByZn6uDpMteQuIf0FzZXeKyoXfFjXo="; - }; - esextractor = fetchFromGitHub { - owner = "Igalia"; - repo = "ESExtractor"; - rev = "v0.2.5"; - hash = "sha256-A3lyTTarR1ZJrXcrLDR5D7H1kBwJNyrPPjEklRM9YBY="; - }; - jsoncpp = fetchFromGitHub { - owner = "open-source-parsers"; - repo = "jsoncpp"; - rev = "9059f5cad030ba11d37818847443a53918c327b1"; - hash = "sha256-m0tz8w8HbtDitx3Qkn3Rxj/XhASiJVkThdeBxIwv3WI="; - }; - glslang = fetchFromGitHub { - owner = "KhronosGroup"; - repo = "glslang"; - rev = "cd2082e0584d4e39d11e3f401184e0d558ab304f"; - hash = "sha256-j7O0j4E8lQ9tqAiuhnD/t6VL45OUvntsoKlhiuCXet4="; - }; - spirv-tools = fetchFromGitHub { - owner = "KhronosGroup"; - repo = "SPIRV-Tools"; - rev = "01828dac778d08f4ebafd2e06bd419f6c84e5984"; - hash = "sha256-i1rDMVpUiNdacDe20DsN67/rzK5V434EzfSv97y+xGU="; - }; - spirv-headers = fetchFromGitHub { - owner = "KhronosGroup"; - repo = "SPIRV-Headers"; - rev = "1feaf4414eb2b353764d01d88f8aa4bcc67b60db"; - hash = "sha256-VOq3r6ZcbDGGxjqC4IoPMGC5n1APUPUAs9xcRzxdyfk="; - }; - video-parser = fetchFromGitHub { - owner = "nvpro-samples"; - repo = "vk_video_samples"; - rev = "7d68747d3524842afaf050c5e00a10f5b8c07904"; - hash = "sha256-L5IYDm0bLq+NlNrzozu0VQx8zL1na6AhrkjZKxOWSnU="; - }; - vulkan-docs = fetchFromGitHub { - owner = "KhronosGroup"; - repo = "Vulkan-Docs"; - rev = "9fff8b252a3688c0231fa78709084bbe677d3bf7"; - hash = "sha256-KpKsKTY5xCSZ5Y92roa0fq/iqc1hVJNS7l87RFcxyRQ="; - }; + # with the vk-cts-sources.py script. + sources = import ./sources.nix { inherit fetchurl fetchFromGitHub; }; in stdenv.mkDerivation (finalAttrs: { pname = "vulkan-cts"; @@ -95,18 +49,12 @@ stdenv.mkDerivation (finalAttrs: { outputs = [ "out" "lib" ]; prePatch = '' - mkdir -p external/ESExtractor external/renderdoc/src external/spirv-headers external/video-parser external/vulkan-docs + mkdir -p external/renderdoc/src cp -r ${renderdoc} external/renderdoc/src/renderdoc_app.h - cp -r ${amber} external/amber/src - cp -r ${esextractor} external/ESExtractor/src - cp -r ${jsoncpp} external/jsoncpp/src - cp -r ${glslang} external/glslang/src - cp -r ${spirv-tools} external/spirv-tools/src - cp -r ${spirv-headers} external/spirv-headers/src - cp -r ${video-parser} external/video-parser/src - cp -r ${vulkan-docs} external/vulkan-docs/src + ${sources.prePatch} + chmod u+w -R external ''; @@ -120,8 +68,6 @@ stdenv.mkDerivation (finalAttrs: { libXau libXdmcp libxcb - spirv-headers - spirv-tools wayland wayland-protocols zlib @@ -129,7 +75,6 @@ stdenv.mkDerivation (finalAttrs: { nativeBuildInputs = [ cmake - glslang makeWrapper ninja pkg-config @@ -159,6 +104,8 @@ stdenv.mkDerivation (finalAttrs: { --add-flags "--deqp-archive-dir=$out/archive-dir" ''; + passthru.updateScript = ./update.sh; + meta = with lib; { description = "Khronos Vulkan Conformance Tests"; homepage = "https://github.com/KhronosGroup/VK-GL-CTS/blob/main/external/vulkancts/README.md"; diff --git a/pkgs/tools/graphics/vulkan-cts/sources.nix b/pkgs/tools/graphics/vulkan-cts/sources.nix new file mode 100644 index 000000000000..e1c6185b5c4b --- /dev/null +++ b/pkgs/tools/graphics/vulkan-cts/sources.nix @@ -0,0 +1,73 @@ +# Autogenerated from vk-cts-sources.py +{ fetchurl, fetchFromGitHub }: +rec { + ESExtractor = fetchFromGitHub { + owner = "Igalia"; + repo = "ESExtractor"; + rev = "v0.3.3"; + hash = "sha256-qqhDv08cLQlLaEj0qfghByK+IohdvQdI2ePfUNFEArQ="; + }; + + amber = fetchFromGitHub { + owner = "google"; + repo = "amber"; + rev = "933ecb4d6288675a92eb1650e0f52b1d7afe8273"; + hash = "sha256-v9z4gv/mTjaCkByZn6uDpMteQuIf0FzZXeKyoXfFjXo="; + }; + + glslang = fetchFromGitHub { + owner = "KhronosGroup"; + repo = "glslang"; + rev = "0bbec2e8f6eca92e925bc589725b108788fc0733"; + hash = "sha256-ZX1WkG2eqDjC/swNBQjG0RvTaijc3U6ehS6qsd19b1g="; + }; + + jsoncpp = fetchFromGitHub { + owner = "open-source-parsers"; + repo = "jsoncpp"; + rev = "9059f5cad030ba11d37818847443a53918c327b1"; + hash = "sha256-m0tz8w8HbtDitx3Qkn3Rxj/XhASiJVkThdeBxIwv3WI="; + }; + + spirv-headers = fetchFromGitHub { + owner = "KhronosGroup"; + repo = "SPIRV-Headers"; + rev = "8e2ad27488ed2f87c068c01a8f5e8979f7086405"; + hash = "sha256-EU69D5+ToET2No2h7MsWuuSwjbl/8TDDDHuVnkUNs/E="; + }; + + spirv-tools = fetchFromGitHub { + owner = "KhronosGroup"; + repo = "SPIRV-Tools"; + rev = "6f276e05ccab210584996bc40a0bef82b91f4f40"; + hash = "sha256-qnFOROpvdUcxohMDlvdiupOoBCoeDUiYtlX8DKbSPgw="; + }; + + video-parser = fetchFromGitHub { + owner = "nvpro-samples"; + repo = "vk_video_samples"; + rev = "7d68747d3524842afaf050c5e00a10f5b8c07904"; + hash = "sha256-L5IYDm0bLq+NlNrzozu0VQx8zL1na6AhrkjZKxOWSnU="; + }; + + vulkan-docs = fetchFromGitHub { + owner = "KhronosGroup"; + repo = "Vulkan-Docs"; + rev = "2f4ef8371aa309f91954536996582066900ef2a0"; + hash = "sha256-uvsDH1emOdw+t/jmqAXG6ssCtLcHZbzAg0SGDxLNrvc="; + }; + + + prePatch = '' + mkdir -p external/ESExtractor external/amber external/glslang external/jsoncpp external/spirv-headers external/spirv-tools external/video-parser external/vulkan-docs + + cp -r ${ESExtractor} external/ESExtractor/src + cp -r ${amber} external/amber/src + cp -r ${glslang} external/glslang/src + cp -r ${jsoncpp} external/jsoncpp/src + cp -r ${spirv-headers} external/spirv-headers/src + cp -r ${spirv-tools} external/spirv-tools/src + cp -r ${video-parser} external/video-parser/src + cp -r ${vulkan-docs} external/vulkan-docs/src + ''; +} diff --git a/pkgs/tools/graphics/vulkan-cts/update.sh b/pkgs/tools/graphics/vulkan-cts/update.sh new file mode 100755 index 000000000000..300fe230264e --- /dev/null +++ b/pkgs/tools/graphics/vulkan-cts/update.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p common-updater-scripts curl jq + +set -euo pipefail + +rawVersion="$(curl ${GITHUB_TOKEN:+" -u \":$GITHUB_TOKEN\""} -s "https://api.github.com/repos/KhronosGroup/VK-GL-CTS/releases" | jq -r 'map(select(.tag_name | startswith("vulkan-cts-"))) | .[0].tag_name')" +basedir="$(git rev-parse --show-toplevel)" + +cd "$basedir" +# Strip prefix +version="$(echo "$rawVersion" | sed 's/vulkan-cts-//')" +update-source-version vulkan-cts "$version" + +# Update imported sources +tmpDir="$(mktemp -d)" +trap "rm -rf $tmpDir" EXIT + +curl -s "https://raw.githubusercontent.com/KhronosGroup/VK-GL-CTS/$rawVersion/external/fetch_sources.py" -o "$tmpDir/fetch_sources.py" +sed -i '/from ctsbuild.common import/d' "$tmpDir/fetch_sources.py" +cd "$(dirname "$0")" +PYTHONPATH="$tmpDir/" ./vk-cts-sources.py diff --git a/pkgs/tools/graphics/vulkan-cts/vk-cts-sources.py b/pkgs/tools/graphics/vulkan-cts/vk-cts-sources.py new file mode 100755 index 000000000000..f3e42bd82e7a --- /dev/null +++ b/pkgs/tools/graphics/vulkan-cts/vk-cts-sources.py @@ -0,0 +1,93 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i python3 -p nix-prefetch-github -p git +#nix-shell -I nixpkgs=../../../../ -i python3 -p "python3.withPackages (ps: with ps; [ nix-prefetch-github ])" -p "git" + +import json +import re +import subprocess +import sys + +import fetch_sources + +def get_github_hash(owner, repo, revision): + result = subprocess.run( + ["nix-prefetch-github", owner, repo, "--json", "--rev", revision], + check=True, + capture_output=True, + text=True, + ) + j = json.loads(result.stdout) + # Remove False values + return {k: v for k, v in j.items() if v} + +def main(): + pkgs = fetch_sources.PACKAGES + pkgs.sort(key = lambda pkg: pkg.baseDir) + existing_sources = {} + + # Fetch hashes from existing sources file + with open("sources.nix") as f: + existing_file = f.read() + + source_re = re.compile("(?P[^ ]+) = fetchFromGitHub[^\n]*\n" + "[^\n]+\n" # owner + "[^\n]+\n" # repo + " *rev = \"(?P[^\"]+)\";\n" + " *hash = \"(?P[^\"]+)\";\n" + ) + + for m in source_re.finditer(existing_file): + if m.group("hash").startswith("sha"): + print(f"Found {m.group('name')}: {m.group('rev')} -> {m.group('hash')}") + existing_sources[m.group("name")] = (m.group("rev"), m.group("hash")) + print() + + + # Write new sources file + with open("sources.nix", "w") as f: + f.write("# Autogenerated from vk-cts-sources.py\n") + f.write("{ fetchurl, fetchFromGitHub }:\n") + f.write("rec {"); + + github_re = re.compile("https://github.com/(?P[^/]+)/(?P[^/]+).git") + + for pkg in pkgs: + if isinstance(pkg, fetch_sources.GitRepo): + ms = github_re.match(pkg.httpsUrl) + + # Check for known hash + hash = None + if pkg.baseDir in existing_sources: + existing_src = existing_sources[pkg.baseDir] + if existing_src[0] == pkg.revision: + hash = existing_src[1] + + if hash is None: + print(f"Fetching {pkg.baseDir}: {pkg.revision}") + hash = get_github_hash(ms.group("owner"), ms.group("repo"), pkg.revision)["hash"] + print(f"Got {pkg.baseDir}: {pkg.revision} -> {hash}") + + f.write(f"\n {pkg.baseDir} = fetchFromGitHub {{\n"); + f.write(f" owner = \"{ms.group('owner')}\";\n"); + f.write(f" repo = \"{ms.group('repo')}\";\n"); + f.write(f" rev = \"{pkg.revision}\";\n"); + f.write(f" hash = \"{hash}\";\n"); + f.write(f" }};\n"); + + f.write("\n\n prePatch = ''\n"); + f.write(" mkdir -p"); + for pkg in pkgs: + if isinstance(pkg, fetch_sources.GitRepo): + f.write(f" external/{pkg.baseDir}") + f.write("\n\n"); + + for pkg in pkgs: + if isinstance(pkg, fetch_sources.GitRepo): + f.write(f" cp -r ${{{pkg.baseDir}}} external/{pkg.baseDir}/{pkg.extractDir}\n"); + + f.write(" '';\n"); + + f.write("}\n"); + +if __name__ == "__main__": + main()