merge: 'issue-6' from email

Before actually getting rate limited, we simply run out of free
sockets with the default limit of just 1024. Bumping this to 16384
helps considerably with derivations that have many
prerequisites (e.g., nixpkgs#texliveFull).

Signed-off-by: Maximilian Marx <mmarx@wh2.tu-dresden.de>

Cargo.lock

@@ -133,6 +133,12 @@ dependencies = [
  "syn 2.0.77",
 ]
 
+[[package]]
+name = "atomic-waker"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
+
 [[package]]
 name = "autocfg"
 version = "1.3.0"
@@ -150,9 +156,9 @@ dependencies = [
  "bitflags 1.3.2",
  "bytes",
  "futures-util",
- "http",
- "http-body",
- "hyper",
+ "http 0.2.12",
+ "http-body 0.4.6",
+ "hyper 0.14.30",
  "itoa",
  "matchit",
  "memchr",
@@ -161,7 +167,7 @@ dependencies = [
  "pin-project-lite",
  "rustversion",
  "serde",
- "sync_wrapper",
+ "sync_wrapper 0.1.2",
  "tower",
  "tower-layer",
  "tower-service",
@@ -176,8 +182,8 @@ dependencies = [
  "async-trait",
  "bytes",
  "futures-util",
- "http",
- "http-body",
+ "http 0.2.12",
+ "http-body 0.4.6",
  "mime",
  "rustversion",
  "tower-layer",
@@ -205,6 +211,12 @@ version = "0.21.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
 
+[[package]]
+name = "base64"
+version = "0.22.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+
 [[package]]
 name = "bitflags"
 version = "1.3.2"
@@ -742,7 +754,26 @@ dependencies = [
  "futures-core",
  "futures-sink",
  "futures-util",
- "http",
+ "http 0.2.12",
+ "indexmap 2.5.0",
+ "slab",
+ "tokio",
+ "tokio-util",
+ "tracing",
+]
+
+[[package]]
+name = "h2"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205"
+dependencies = [
+ "atomic-waker",
+ "bytes",
+ "fnv",
+ "futures-core",
+ "futures-sink",
+ "http 1.1.0",
  "indexmap 2.5.0",
  "slab",
  "tokio",
@@ -768,7 +799,7 @@ version = "7.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d"
 dependencies = [
- "base64",
+ "base64 0.21.7",
  "byteorder",
  "flate2",
  "nom",
@@ -812,6 +843,17 @@ dependencies = [
  "itoa",
 ]
 
+[[package]]
+name = "http"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258"
+dependencies = [
+ "bytes",
+ "fnv",
+ "itoa",
+]
+
 [[package]]
 name = "http-body"
 version = "0.4.6"
@@ -819,7 +861,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2"
 dependencies = [
  "bytes",
- "http",
+ "http 0.2.12",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "http-body"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
+dependencies = [
+ "bytes",
+ "http 1.1.0",
+]
+
+[[package]]
+name = "http-body-util"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f"
+dependencies = [
+ "bytes",
+ "futures-util",
+ "http 1.1.0",
+ "http-body 1.0.1",
  "pin-project-lite",
 ]
 
@@ -851,9 +916,9 @@ dependencies = [
  "futures-channel",
  "futures-core",
  "futures-util",
- "h2",
- "http",
- "http-body",
+ "h2 0.3.26",
+ "http 0.2.12",
+ "http-body 0.4.6",
  "httparse",
  "httpdate",
  "itoa",
@@ -865,13 +930,50 @@ dependencies = [
  "want",
 ]
 
+[[package]]
+name = "hyper"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a"
+dependencies = [
+ "bytes",
+ "futures-channel",
+ "futures-util",
+ "h2 0.4.6",
+ "http 1.1.0",
+ "http-body 1.0.1",
+ "httparse",
+ "itoa",
+ "pin-project-lite",
+ "smallvec",
+ "tokio",
+ "want",
+]
+
+[[package]]
+name = "hyper-rustls"
+version = "0.27.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333"
+dependencies = [
+ "futures-util",
+ "http 1.1.0",
+ "hyper 1.5.0",
+ "hyper-util",
+ "rustls",
+ "rustls-pki-types",
+ "tokio",
+ "tokio-rustls",
+ "tower-service",
+]
+
 [[package]]
 name = "hyper-timeout"
 version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1"
 dependencies = [
- "hyper",
+ "hyper 0.14.30",
  "pin-project-lite",
  "tokio",
  "tokio-io-timeout",
@@ -879,15 +981,37 @@ dependencies = [
 
 [[package]]
 name = "hyper-tls"
-version = "0.5.0"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
+checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0"
 dependencies = [
  "bytes",
- "hyper",
+ "http-body-util",
+ "hyper 1.5.0",
+ "hyper-util",
  "native-tls",
  "tokio",
  "tokio-native-tls",
+ "tower-service",
+]
+
+[[package]]
+name = "hyper-util"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b"
+dependencies = [
+ "bytes",
+ "futures-channel",
+ "futures-util",
+ "http 1.1.0",
+ "http-body 1.0.1",
+ "hyper 1.5.0",
+ "pin-project-lite",
+ "socket2",
+ "tokio",
+ "tower-service",
+ "tracing",
 ]
 
 [[package]]
@@ -1125,6 +1249,7 @@ dependencies = [
  "pretty_env_logger",
  "rayon",
  "reqwest",
+ "rlimit",
  "scraper",
  "serde",
  "serde_json",
@@ -1559,20 +1684,24 @@ checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
 
 [[package]]
 name = "reqwest"
-version = "0.11.27"
+version = "0.12.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62"
+checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b"
 dependencies = [
- "base64",
+ "base64 0.22.1",
  "bytes",
  "encoding_rs",
+ "futures-channel",
  "futures-core",
  "futures-util",
- "h2",
- "http",
- "http-body",
- "hyper",
+ "h2 0.4.6",
+ "http 1.1.0",
+ "http-body 1.0.1",
+ "http-body-util",
+ "hyper 1.5.0",
+ "hyper-rustls",
  "hyper-tls",
+ "hyper-util",
  "ipnet",
  "js-sys",
  "log",
@@ -1585,7 +1714,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_urlencoded",
- "sync_wrapper",
+ "sync_wrapper 1.0.1",
  "system-configuration",
  "tokio",
  "tokio-native-tls",
@@ -1594,7 +1723,31 @@ dependencies = [
  "wasm-bindgen",
  "wasm-bindgen-futures",
  "web-sys",
- "winreg",
+ "windows-registry",
+]
+
+[[package]]
+name = "ring"
+version = "0.17.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
+dependencies = [
+ "cc",
+ "cfg-if",
+ "getrandom",
+ "libc",
+ "spin",
+ "untrusted",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "rlimit"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7043b63bd0cd1aaa628e476b80e6d4023a3b50eb32789f2728908107bd0c793a"
+dependencies = [
+ "libc",
 ]
 
 [[package]]
@@ -1623,12 +1776,42 @@ dependencies = [
 ]
 
 [[package]]
-name = "rustls-pemfile"
-version = "1.0.4"
+name = "rustls"
+version = "0.23.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c"
+checksum = "5fbb44d7acc4e873d613422379f69f237a1b141928c02f6bc6ccfddddc2d7993"
 dependencies = [
- "base64",
+ "once_cell",
+ "rustls-pki-types",
+ "rustls-webpki",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
+name = "rustls-pemfile"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50"
+dependencies = [
+ "rustls-pki-types",
+]
+
+[[package]]
+name = "rustls-pki-types"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b"
+
+[[package]]
+name = "rustls-webpki"
+version = "0.102.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9"
+dependencies = [
+ "ring",
+ "rustls-pki-types",
+ "untrusted",
 ]
 
 [[package]]
@@ -1824,6 +2007,12 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "spin"
+version = "0.9.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
+
 [[package]]
 name = "stable_deref_trait"
 version = "1.2.0"
@@ -1862,6 +2051,12 @@ version = "0.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
 
+[[package]]
+name = "subtle"
+version = "2.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
+
 [[package]]
 name = "syn"
 version = "1.0.109"
@@ -1891,21 +2086,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
 
 [[package]]
-name = "system-configuration"
-version = "0.5.1"
+name = "sync_wrapper"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7"
+checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394"
 dependencies = [
- "bitflags 1.3.2",
+ "futures-core",
+]
+
+[[package]]
+name = "system-configuration"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b"
+dependencies = [
+ "bitflags 2.6.0",
  "core-foundation",
  "system-configuration-sys",
 ]
 
 [[package]]
 name = "system-configuration-sys"
-version = "0.5.0"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9"
+checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4"
 dependencies = [
  "core-foundation-sys",
  "libc",
@@ -2038,6 +2242,17 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "tokio-rustls"
+version = "0.26.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4"
+dependencies = [
+ "rustls",
+ "rustls-pki-types",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-stream"
 version = "0.1.15"
@@ -2071,12 +2286,12 @@ dependencies = [
  "async-stream",
  "async-trait",
  "axum",
- "base64",
+ "base64 0.21.7",
  "bytes",
- "h2",
- "http",
- "http-body",
- "hyper",
+ "h2 0.3.26",
+ "http 0.2.12",
+ "http-body 0.4.6",
+ "hyper 0.14.30",
  "hyper-timeout",
  "percent-encoding",
  "pin-project",
@@ -2201,6 +2416,12 @@ version = "0.1.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d"
 
+[[package]]
+name = "untrusted"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
+
 [[package]]
 name = "url"
 version = "2.5.2"
@@ -2340,7 +2561,37 @@ version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
 dependencies = [
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
+]
+
+[[package]]
+name = "windows-registry"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0"
+dependencies = [
+ "windows-result",
+ "windows-strings",
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-result"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-strings"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"
+dependencies = [
+ "windows-result",
+ "windows-targets 0.52.6",
 ]
 
 [[package]]
@@ -2491,16 +2742,6 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
-[[package]]
-name = "winreg"
-version = "0.50.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1"
-dependencies = [
- "cfg-if",
- "windows-sys 0.48.0",
-]
-
 [[package]]
 name = "workspace-hack"
 version = "0.1.0"
@@ -2533,3 +2774,9 @@ dependencies = [
  "quote",
  "syn 2.0.77",
 ]
+
+[[package]]
+name = "zeroize"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"

crates/nix-weather/Cargo.toml

@@ -27,7 +27,8 @@ log = "0.4.21"
 openssl = { version = "0.10.63" }
 pretty_env_logger = "0.5.0"
 rayon = "1.9.0"
-reqwest = { version = "0.11.24", features = ["blocking"] }
+reqwest = { version = "0.12", features = ["blocking"] }
+rlimit = "0.10.2"
 scraper = "0.18.1"
 serde = "1.0.197"
 serde_json = "1.0.114"

crates/nix-weather/src/cli.rs

@@ -1,9 +1,10 @@
 // SPDX-FileCopyrightText: 2023-2024 Christina Sørensen
 // SPDX-FileContributor: Christina Sørensen
+// SPDX-FileContributor: Maximilian Marx
 //
 // SPDX-License-Identifier: EUPL-1.2
 
-use clap::{arg, command, crate_authors, value_parser, ArgAction, Command};
+use clap::{arg, command, crate_authors, value_parser, ArgAction, ArgGroup, Command};
 
 const DEFAULT_CACHE: &str = "cache.nixos.org";
 
@@ -38,4 +39,11 @@ pub fn build_cli() -> Command {
         .long("print-build-logs")
         .conflicts_with("verbose"),
     )
+    .arg(arg!(-'4' --"only-ipv4" "Use IPv4 addresses only.").action(ArgAction::SetTrue))
+    .arg(arg!(-'6' --"only-ipv6" "Use IPv6 addresses only.").action(ArgAction::SetTrue))
+    .group(
+      ArgGroup::new("address_family")
+        .args(["only-ipv4", "only-ipv6"])
+        .required(false),
+    )
 }

crates/nix-weather/src/main.rs

@@ -1,15 +1,17 @@
 // SPDX-FileCopyrightText: 2024 Christina Sørensen
 // SPDX-FileContributor: Christina Sørensen
+// SPDX-FileContributor: Maximilian Marx
 //
 // SPDX-License-Identifier: EUPL-1.2
 
+use std::sync::Arc;
 use std::time::Instant;
 use std::{env, io, net::SocketAddr};
 
-use dns_lookup::lookup_host;
 use futures::future::join_all;
 use gethostname::gethostname;
 use itertools::Itertools;
+use net::AddressFamilyFilter;
 
 use crate::nix::get_requisites;
 
@@ -20,6 +22,9 @@ mod nix;
 /// The initial time to wait on http 104, in milliseconds
 const SLIDE: u64 = 100;
 
+// Open files limit to try to set
+const NOFILES_LIMIT: u64 = 16384;
+
 const DEFAULT_CACHE: &str = "cache.nixos.org";
 const DEFAULT_CONFIG_DIR: &str = "/etc/nixos";
 
@@ -91,14 +96,34 @@ async fn main() -> io::Result<()> {
     config_dir = DEFAULT_CONFIG_DIR.to_string();
   }
 
+  let address_family_filter = if matches.get_flag("only-ipv4") {
+    AddressFamilyFilter::OnlyIPv4
+  } else if matches.get_flag("only-ipv6") {
+    AddressFamilyFilter::OnlyIPv6
+  } else {
+    Default::default()
+  };
+
   let domain = cache_url.to_owned();
-  let ips: Vec<std::net::IpAddr> = lookup_host(&domain).unwrap();
+  let ips: Vec<std::net::IpAddr> = address_family_filter
+    .lookup_host(&domain)
+    .unwrap()
+    .collect();
 
   log::debug!("{:#?}", &ips);
 
+  // try to increase NOFILES runtime limit
+  if rlimit::increase_nofile_limit(NOFILES_LIMIT).is_err() {
+    log::warn!(
+      "Failed to increase NOFILES limit, still at {:#?}",
+      rlimit::Resource::NOFILE.get().unwrap_or_default()
+    );
+  }
+
   let domain_addr = SocketAddr::new(ips[0], 443);
 
   let client = reqwest::Client::builder()
+    .dns_resolver(Arc::new(address_family_filter))
     .resolve(&domain, domain_addr)
     .build()
     .unwrap();

crates/nix-weather/src/net.rs

@@ -1,15 +1,59 @@
 // SPDX-FileCopyrightText: 2024 Christina Sørensen
 // SPDX-FileContributor: Christina Sørensen
+// SPDX-FileContributor: Maximilian Marx
 //
 // SPDX-License-Identifier: EUPL-1.2
 
-use std::time::Duration;
+use std::{
+  io,
+  net::{IpAddr, SocketAddr},
+  time::Duration,
+};
 
-use reqwest::Client;
+use reqwest::{dns::Resolve, Client};
 use tokio::time::sleep;
 
 const MAX_SLIDE: u64 = 1000;
 
+#[derive(Debug, Copy, Clone, Default)]
+pub enum AddressFamilyFilter {
+  #[default]
+  Both,
+  OnlyIPv4,
+  OnlyIPv6,
+}
+
+impl AddressFamilyFilter {
+  pub fn lookup_host(self, host: &str) -> io::Result<impl Iterator<Item = IpAddr>> {
+    let addresses = dns_lookup::lookup_host(host)?;
+    Ok(self.filter_addresses(addresses))
+  }
+
+  fn filter_addresses<T>(self, addresses: T) -> impl Iterator<Item = IpAddr>
+  where
+    T: IntoIterator<Item = IpAddr>,
+  {
+    addresses.into_iter().filter(move |address| match self {
+      Self::Both => true,
+      Self::OnlyIPv4 => matches!(address, IpAddr::V4(_)),
+      Self::OnlyIPv6 => matches!(address, IpAddr::V6(_)),
+    })
+  }
+}
+
+impl Resolve for AddressFamilyFilter {
+  fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving {
+    let filter = *self;
+    Box::pin(async move {
+      let addresses = filter.lookup_host(name.as_str())?;
+      let socket_addresses: Box<dyn Iterator<Item = SocketAddr> + Send> =
+        Box::new(addresses.map(|ip| SocketAddr::new(ip, 0)));
+
+      Ok(socket_addresses)
+    })
+  }
+}
+
 pub async fn nar_exists(client: Client, domain: &str, hash: &str, slide: u64) -> usize {
   let response = client
     .head(format!("https://{domain}/{hash}.narinfo"))
@@ -19,10 +63,31 @@ pub async fn nar_exists(client: Client, domain: &str, hash: &str, slide: u64) ->
   match response {
     Ok(response) if response.status().as_u16() == 200 => 1,
     Ok(response) if response.status().as_u16() == 404 => 0,
-    _ => {
+    Ok(response) if response.status().as_u16() == 429 => {
       // We're so fast now we get rate limited.
-      //
-      // Writng an actual sliding window seems kinda hard,
+
+      let wait = if let Some(retry_after) = response.headers().get("Retry-After") {
+        if let Ok(seconds) = String::from_utf8_lossy(retry_after.as_bytes()).parse::<u64>() {
+          seconds * 1000
+        } else {
+          slide
+        }
+      } else {
+        slide
+      };
+
+      sleep(Duration::from_millis(wait)).await;
+      Box::pin(nar_exists(
+        client,
+        domain,
+        hash,
+        std::cmp::min(slide * 2, MAX_SLIDE),
+      ))
+      .await
+    }
+    _ => {
+      // We might also be out of sockets, so let's wait on that
+      // Writing an actual sliding window seems kinda hard,
       // so we do this instead.
       log::trace!("rate limited! {slide}");
       sleep(Duration::from_millis(slide)).await;

crates/workspace-hack/Cargo.toml

@@ -1,6 +1,7 @@
 # SPDX-FileCopyrightText: 2024 Christina Sørensen
 #
 # SPDX-License-Identifier: EUPL-1.2
+# SPDX-FileContributor: Maximilian Marx
 
 # This file is generated by `cargo hakari`.
 # To regenerate, run:
@@ -12,6 +13,7 @@ version = "0.1.0"
 description = "workspace-hack package, managed by hakari"
 # You can choose to publish this crate: see https://docs.rs/cargo-hakari/latest/cargo_hakari/publishing.
 publish = false
+edition.workspace = true
 
 # The parts of the file between the BEGIN HAKARI SECTION and END HAKARI SECTION comments
 # are managed by hakari.