diff --git a/pkgs/servers/invidious/default.nix b/pkgs/servers/invidious/default.nix index 0b1cea5fd689..3f5bb42a48f2 100644 --- a/pkgs/servers/invidious/default.nix +++ b/pkgs/servers/invidious/default.nix @@ -1,11 +1,11 @@ -{ lib, stdenv, crystal, fetchFromGitea, librsvg, pkg-config, libxml2, openssl, shards, sqlite, lsquic, videojs, nixosTests }: +{ lib, stdenv, crystal, fetchFromGitea, librsvg, pkg-config, libxml2, openssl, shards, sqlite, videojs, nixosTests }: let # All versions, revisions, and checksums are stored in ./versions.json. # The update process is the following: # * pick the latest commit - # * update .invidious.rev, .invidious.version, and .invidious.sha256 + # * update .invidious.rev, .invidious.version, and .invidious.hash # * prefetch the videojs dependencies with scripts/fetch-player-dependencies.cr - # and update .videojs.sha256 (they are normally fetched during build + # and update .videojs.hash (they are normally fetched during build # but nix's sandboxing does not allow that) # * if shard.lock changed # * recreate shards.nix by running crystal2nix @@ -23,7 +23,7 @@ crystal.buildCrystalPackage rec { owner = "iv-org"; repo = pname; fetchSubmodules = true; - inherit (versions.invidious) rev sha256; + inherit (versions.invidious) rev hash; }; postPatch = @@ -45,7 +45,7 @@ crystal.buildCrystalPackage rec { substituteInPlace src/invidious.cr \ --replace ${lib.escapeShellArg branchTemplate} '"master"' \ --replace ${lib.escapeShellArg commitTemplate} '"${lib.substring 0 7 versions.invidious.rev}"' \ - --replace ${lib.escapeShellArg versionTemplate} '"${lib.replaceStrings ["-"] ["."] (lib.substring 9 10 version)}"' \ + --replace ${lib.escapeShellArg versionTemplate} '"${lib.concatStringsSep "." (lib.drop 2 (lib.splitString "-" version))}"' \ --replace ${lib.escapeShellArg assetCommitTemplate} '"${lib.substring 0 7 versions.invidious.rev}"' # Patch the assets and locales paths to be absolute @@ -75,20 +75,9 @@ crystal.buildCrystalPackage rec { "--verbose" "--no-debug" "-Dskip_videojs_download" - "-Ddisable_quic" ]; }; - postConfigure = '' - # lib includes nix store paths which can’t be patched, so the links have to - # be dereferenced first. - cp -rL lib lib2 - rm -r lib - mv lib2 lib - chmod +w -R lib - cp ${lsquic}/lib/liblsquic.a lib/lsquic/src/lsquic/ext - ''; - postInstall = '' mkdir -p $out/share/invidious/config @@ -102,15 +91,16 @@ crystal.buildCrystalPackage rec { # environment variable. Even though the database and hmac_key are # bogus, --help still works. installCheckPhase = '' - INVIDIOUS_CONFIG="$(cat < $out - ''; - }; - - # lsquic requires a specific boringssl version (noted in its README) - boringssl' = boringssl.overrideAttrs ({ preBuild, ... }: { - version = versions.boringssl.rev; - src = fetchgit { - url = "https://boringssl.googlesource.com/boringssl"; - inherit (versions.boringssl) rev sha256; - }; - - patches = [ - # Use /etc/ssl/certs/ca-certificates.crt instead of /etc/ssl/cert.pem - ./use-etc-ssl-certs.patch - - # because lsquic requires that specific boringssl version and that - # version does not yet include fixes for gcc11 build errors, they - # must be backported - (fetchGitilesPatch { - name = "fix-mismatch-between-header-and-implementation-of-bn_sqr_comba8.patch"; - url = "https://boringssl.googlesource.com/boringssl/+/139adff9b27eaf0bdaac664ec4c9a7db2fe3f920"; - sha256 = "05sp602dvh50v46jkzmh4sf4wqnq5bwy553596g2rhxg75bailjj"; - }) - (fetchGitilesPatch { - name = "use-an-unsized-helper-for-truncated-SHA-512-variants.patch"; - url = "https://boringssl.googlesource.com/boringssl/+/a24ab549e6ae246b391155d7bed3790ac0e07de2"; - sha256 = "0483jkpg4g64v23ln2blb74xnmzdjcn3r7w4zk7nfg8j3q5f9lxm"; - }) -/* - # the following patch is too complex, so we will modify the build flags - # of crypto/fipsmodule/CMakeFiles/fipsmodule.dir/bcm.c.o in preBuild - # and turn off -Werror=stringop-overflow - (fetchGitilesPatch { - name = "make-md32_common.h-single-included-and-use-an-unsized-helper-for-SHA-256.patch"; - url = "https://boringssl.googlesource.com/boringssl/+/597ffef971dd980b7de5e97a0c9b7ca26eec94bc"; - sha256 = "1y0bkkdf1ccd6crx326agp01q22clm4ai4p982y7r6dkmxmh52qr"; - }) -*/ - (fetchGitilesPatch { - name = "fix-array-parameter-warnings.patch"; - url = "https://boringssl.googlesource.com/boringssl/+/92c6fbfc4c44dc8462d260d836020d2b793e7804"; - sha256 = "0h4sl95i8b0dj0na4ngf50wg54raxyjxl1zzwdc810abglp10vnv"; - }) - ]; - - preBuild = preBuild + lib.optionalString stdenv.isLinux '' - sed -e '/^build crypto\/fipsmodule\/CMakeFiles\/fipsmodule\.dir\/bcm\.c\.o:/,/^ *FLAGS =/ s/^ *FLAGS = -Werror/& -Wno-error=stringop-overflow/' \ - -i build.ninja - '' + lib.optionalString stdenv.cc.isGNU '' - # Silence warning that causes build failures with GCC. - sed -e '/^build ssl\/test\/CMakeFiles\/bssl_shim\.dir\/settings_writer\.cc\.o:/,/^ *FLAGS =/ s/^ *FLAGS = -Werror/& -Wno-error=ignored-attributes/' \ - -e '/^build ssl\/test\/CMakeFiles\/handshaker\.dir\/settings_writer\.cc\.o:/,/^ *FLAGS =/ s/^ *FLAGS = -Werror/& -Wno-error=ignored-attributes/' \ - -i build.ninja - '' + lib.optionalString stdenv.cc.isClang ( - # Silence warnings that cause build failures with newer versions of clang. - let - clangVersion = lib.getVersion stdenv.cc; - in - lib.optionalString (lib.versionAtLeast clangVersion "13") '' - sed -e '/^build crypto\/CMakeFiles\/crypto\.dir\/x509\/t_x509\.c\.o:/,/^ *FLAGS =/ s/^ *FLAGS = -Werror/& -Wno-error=unused-but-set-variable/' \ - -e '/^build tool\/CMakeFiles\/bssl\.dir\/digest\.cc\.o:/,/^ *FLAGS =/ s/^ *FLAGS = -Werror/& -Wno-error=unused-but-set-variable/' \ - -i build.ninja - '' + lib.optionalString (lib.versionAtLeast clangVersion "16") '' - sed -e '/^build crypto\/CMakeFiles\/crypto\.dir\/trust_token\/trust_token\.c\.o:/,/^ *FLAGS =/ s/^ *FLAGS = -Werror/& -Wno-error=single-bit-bitfield-constant-conversion/' \ - -i build.ninja - '' - ); - }); -in -stdenv.mkDerivation rec { - pname = "lsquic"; - version = versions.lsquic.version; - - src = fetchFromGitHub { - owner = "litespeedtech"; - repo = pname; - rev = "v${version}"; - inherit (versions.lsquic) sha256; - fetchSubmodules = true; - }; - - postPatch = '' - substituteInPlace CMakeLists.txt \ - --replace ".so" "${stdenv.hostPlatform.extensions.sharedLibrary}" - ''; - - nativeBuildInputs = [ cmake perl ]; - buildInputs = [ boringssl' libevent zlib ]; - - cmakeFlags = [ - "-DBORINGSSL_DIR=${lib.getDev boringssl'}" - "-DBORINGSSL_LIB_crypto=${lib.getLib boringssl'}/lib/libcrypto.a" - "-DBORINGSSL_LIB_ssl=${lib.getLib boringssl'}/lib/libssl.a" - "-DZLIB_LIB=${zlib}/lib/libz.so" - ]; - - # adapted from lsquic.cr’s Dockerfile - # (https://github.com/iv-org/lsquic.cr/blob/master/docker/Dockerfile) - installPhase = '' - runHook preInstall - - mkdir combinedlib - cd combinedlib - ar -x ${lib.getLib boringssl'}/lib/libssl.a - ar -x ${lib.getLib boringssl'}/lib/libcrypto.a - ar -x ../src/liblsquic/liblsquic.a - ar rc liblsquic.a *.o - ranlib liblsquic.a - install -D liblsquic.a $out/lib/liblsquic.a - - runHook postInstall - ''; - - passthru.boringssl = boringssl'; - - meta = with lib; { - description = "A library for QUIC and HTTP/3 (version for Invidious)"; - homepage = "https://github.com/litespeedtech/lsquic"; - maintainers = with maintainers; [ infinisil sbruder ]; - license = with licenses; [ openssl isc mit bsd3 ]; # statically links against boringssl, so has to include its licenses - }; -} diff --git a/pkgs/servers/invidious/shards.nix b/pkgs/servers/invidious/shards.nix index e5f297d902c0..8e38d563be20 100644 --- a/pkgs/servers/invidious/shards.nix +++ b/pkgs/servers/invidious/shards.nix @@ -1,80 +1,62 @@ { + ameba = { + url = "https://github.com/crystal-ameba/ameba.git"; + rev = "v1.5.0"; + sha256 = "1idivsbpmi40aqvs82fsv37nrgikirprxrj3ls9chsb876fq9p2d"; + }; athena-negotiation = { - owner = "athena-framework"; - repo = "negotiation"; + url = "https://github.com/athena-framework/negotiation.git"; rev = "v0.1.1"; sha256 = "1vkk59lqrxb0l8kyzs114i3c18zb2bdiah2xhazkk8q7x6fz4yzk"; }; backtracer = { - owner = "sija"; - repo = "backtracer.cr"; + url = "https://github.com/sija/backtracer.cr.git"; rev = "v1.2.1"; sha256 = "02r1l7rn2wsljkx495s5s7j04zgn73m2kx0hkzs7620camvlwbqq"; }; db = { - owner = "crystal-lang"; - repo = "crystal-db"; + url = "https://github.com/crystal-lang/crystal-db.git"; rev = "v0.10.1"; sha256 = "03c5h14z6h2mxnx949lihnyqjd19hcj38iasdwq9fp95h8cld376"; }; exception_page = { - owner = "crystal-loot"; - repo = "exception_page"; + url = "https://github.com/crystal-loot/exception_page.git"; rev = "v0.2.2"; sha256 = "1c8askb9b7621jjz5pjj6b8pdbhw3r1l3dym6swg1saspf5j3jwi"; }; kemal = { - owner = "kemalcr"; - repo = "kemal"; + url = "https://github.com/kemalcr/kemal.git"; rev = "v1.1.2"; sha256 = "1149q4qw0zrws5asqqr4snrdi67xsmisdcq58zcrbgqgsxgly9d0"; }; kilt = { - owner = "jeromegn"; - repo = "kilt"; + url = "https://github.com/jeromegn/kilt.git"; rev = "v0.6.1"; sha256 = "0dpc15y9m8c5l9zdfif6jlf7zmkrlm9w4m2igi5xa22fdjwamwfp"; }; - lsquic = { - owner = "iv-org"; - repo = "lsquic.cr"; - rev = "v2.18.1-2"; - sha256 = "0bljk0pwbjb813dfwrhgi00w2ai09k868xvak4hfzdkbmpc7id6y"; - }; pg = { - owner = "will"; - repo = "crystal-pg"; + url = "https://github.com/will/crystal-pg.git"; rev = "v0.24.0"; sha256 = "07i5bqkv5j6y6f8v5cpqdxc5wzzrvgv3ds24znv4mzv6nc84csn4"; }; protodec = { - owner = "iv-org"; - repo = "protodec"; - rev = "v0.1.4"; - sha256 = "15azh9izxqgwpgkpicmivfdz31wkibnwy09rwhxsg0lyc4wf8xj9"; + url = "https://github.com/iv-org/protodec.git"; + rev = "v0.1.5"; + sha256 = "09cm36skv2mxqrlczp0j1g7cf8wsfdqr8q39nxyj3ggc3yadp8bc"; }; radix = { - owner = "luislavena"; - repo = "radix"; + url = "https://github.com/luislavena/radix.git"; rev = "v0.4.1"; sha256 = "1l08cydkdidq9yyil1wl240hvk41iycv04jrg6nx5mkvzw4z1bzg"; }; spectator = { - owner = "icy-arctic-fox"; - repo = "spectator"; + url = "https://github.com/icy-arctic-fox/spectator.git"; rev = "v0.10.4"; sha256 = "0rcxq2nbslvwrd8m9ajw6dzaw3hagxmkdy9s8p34cgnr4c9dijdq"; }; sqlite3 = { - owner = "crystal-lang"; - repo = "crystal-sqlite3"; + url = "https://github.com/crystal-lang/crystal-sqlite3.git"; rev = "v0.18.0"; sha256 = "03nnvpchhq9f9ywsm3pk2rrj4a3figw7xs96zdziwgr5znkz6x93"; }; - ameba = { - owner = "crystal-ameba"; - repo = "ameba"; - rev = "v0.14.3"; - sha256 = "1cfr95xi6hsyxw1wlrh571hc775xhwmssk3k14i8b7dgbwfmm5x1"; - }; } diff --git a/pkgs/servers/invidious/update.sh b/pkgs/servers/invidious/update.sh index d7302e16bfbd..82c8186bebc4 100755 --- a/pkgs/servers/invidious/update.sh +++ b/pkgs/servers/invidious/update.sh @@ -35,13 +35,14 @@ if [ ! -d "$git_dir" ]; then git init --initial-branch="$git_branch" "$git_dir" git -C "$git_dir" remote add origin "$git_url" fi -git -C "$git_dir" fetch origin "$git_branch" +git -C "$git_dir" fetch origin --tags "$git_branch" # use latest commit before today, we should not call the version *today* # because there might still be commits coming # use the day of the latest commit we picked as version new_rev=$(git -C "$git_dir" log -n 1 --format='format:%H' --before="${today}T00:00:00Z" "origin/$git_branch") -new_version="unstable-$(TZ=UTC git -C "$git_dir" log -n 1 --date='format-local:%Y-%m-%d' --format='%cd' "$new_rev")" +new_tag=$(git -C "$git_dir" describe --tags --abbrev=0 "$new_rev") +new_version="$new_tag-unstable-$(TZ=UTC git -C "$git_dir" log -n 1 --date='format-local:%Y-%m-%d' --format='%cd' "$new_rev")" info "latest commit before $today: $new_rev" if [ "$new_rev" = "$old_rev" ]; then @@ -51,8 +52,8 @@ fi json_set '.invidious.version' "$new_version" json_set '.invidious.rev' "$new_rev" -new_sha256=$(nix-prefetch -I 'nixpkgs=../../..' "$pkg") -json_set '.invidious.sha256' "$new_sha256" +new_hash=$(nix-prefetch -I 'nixpkgs=../../..' "$pkg") +json_set '.invidious.hash' "$new_hash" commit_msg="$pkg: $old_version -> $new_version" # fetch video.js dependencies @@ -60,37 +61,14 @@ info "Running scripts/fetch-player-dependencies.cr..." git -C "$git_dir" reset --hard "$new_rev" (cd "$git_dir" && crystal run scripts/fetch-player-dependencies.cr -- --minified) rm -f "$git_dir/assets/videojs/.gitignore" -videojs_new_sha256=$(nix-hash --type sha256 --base32 "$git_dir/assets/videojs") -json_set '.videojs.sha256' "$videojs_new_sha256" +videojs_new_hash=$(nix-hash --type sha256 --sri "$git_dir/assets/videojs") +json_set '.videojs.hash' "$videojs_new_hash" if git -C "$git_dir" diff-tree --quiet "${old_rev}..${new_rev}" -- 'shard.lock'; then info "shard.lock did not change since $old_rev." else info "Updating shards.nix..." crystal2nix -- "$git_dir/shard.lock" # argv's index seems broken - - lsquic_old_version=$(json_get '.lsquic.version') - # lsquic.cr's version tracks lsquic's, so lsquic must be updated to the - # version in the shards file - lsquic_new_version=$(nix eval --raw -f 'shards.nix' lsquic.rev \ - | sed -e 's/^v//' -e 's/-[0-9]*$//') - if [ "$lsquic_old_version" != "$lsquic_new_version" ]; then - info "Updating lsquic to $lsquic_new_version..." - json_set '.lsquic.version' "$lsquic_new_version" - lsquic_new_sha256=$(nix-prefetch -I 'nixpkgs=../../..' "${pkg}.lsquic") - json_set '.lsquic.sha256' "$lsquic_new_sha256" - - info "Updating boringssl..." - # lsquic specifies the boringssl commit it requires in its README - boringssl_new_rev=$(curl -LSsf "https://github.com/litespeedtech/lsquic/raw/v${lsquic_new_version}/README.md" \ - | grep -Pom1 '(?<=^git checkout ).*') - json_set '.boringssl.rev' "$boringssl_new_rev" - boringssl_new_sha256=$(nix-prefetch -I 'nixpkgs=../../..' "${pkg}.lsquic.boringssl") - json_set '.boringssl.sha256' "$boringssl_new_sha256" - commit_msg="$commit_msg - -lsquic: $lsquic_old_version -> $lsquic_new_version" - fi fi git commit --verbose --message "$commit_msg" -- versions.json shards.nix diff --git a/pkgs/servers/invidious/versions.json b/pkgs/servers/invidious/versions.json index 73915248c706..38a59c02413c 100644 --- a/pkgs/servers/invidious/versions.json +++ b/pkgs/servers/invidious/versions.json @@ -1,18 +1,10 @@ { - "boringssl": { - "rev": "251b5169fd44345f455438312ec4e18ae07fd58c", - "sha256": "sha256-EU6T9yQCdOLx98Io8o01rEsgxDFF/Xoy42LgPopD2/A=" - }, "invidious": { - "rev": "c005ada48723808e507d0a4d5a3363a1c14a4f07", - "sha256": "sha256-KbnBdAAjScwKV4uUzyBXAQx2C7MqCdCM3gSvgNIzKAU=", - "version": "unstable-2024-01-29" - }, - "lsquic": { - "sha256": "sha256-hG8cUvhbCNeMOsKkaJlgGpzUrIx47E/WhmPIdI5F3qM=", - "version": "2.18.1" + "rev": "e8a36985aff1a5b33ddf9abea85dd2c23422c2f7", + "hash": "sha256-3nU6z1rd1oiNmIz3Ok02xBsT4oNSGX/n+3/WbRVCbhI=", + "version": "0.20.1-unstable-2024-02-18" }, "videojs": { - "sha256": "0zj8fgxdg6jsllaxn795xipa6yxh4yf08hb8x0idyg74q37gfh4c" + "hash": "sha256-jED3zsDkPN8i6GhBBJwnsHujbuwlHdsVpVqa1/pzSH4=" } } diff --git a/pkgs/servers/invidious/videojs.nix b/pkgs/servers/invidious/videojs.nix index 4016f8e1258d..35ad0b1f0655 100644 --- a/pkgs/servers/invidious/videojs.nix +++ b/pkgs/servers/invidious/videojs.nix @@ -14,5 +14,5 @@ stdenvNoCC.mkDerivation { outputHashAlgo = "sha256"; outputHashMode = "recursive"; - outputHash = versions.videojs.sha256; + outputHash = versions.videojs.hash; } diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 4465b0e58712..1a88014658eb 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -9398,8 +9398,6 @@ with pkgs; internetarchive = with python3Packages; toPythonApplication internetarchive; invidious = callPackage ../servers/invidious { - # needs a specific version of lsquic - lsquic = callPackage ../servers/invidious/lsquic.nix { }; # normally video.js is downloaded at build time videojs = callPackage ../servers/invidious/videojs.nix { }; };