merge: 'issue-14' from email
All checks were successful
conventional commits / conventional commits (push) Successful in 6s
build / run (push) Successful in 4m39s
check / run (push) Successful in 49s

This commit is contained in:
Christina Sørensen 2024-10-27 15:36:05 +01:00
commit 4324ee17e8
Signed by: cafkafk
GPG key ID: 26C542FD97F965CE
6 changed files with 362 additions and 57 deletions

339
Cargo.lock generated
View file

@ -133,6 +133,12 @@ dependencies = [
"syn 2.0.77",
]
[[package]]
name = "atomic-waker"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
[[package]]
name = "autocfg"
version = "1.3.0"
@ -150,9 +156,9 @@ dependencies = [
"bitflags 1.3.2",
"bytes",
"futures-util",
"http",
"http-body",
"hyper",
"http 0.2.12",
"http-body 0.4.6",
"hyper 0.14.30",
"itoa",
"matchit",
"memchr",
@ -161,7 +167,7 @@ dependencies = [
"pin-project-lite",
"rustversion",
"serde",
"sync_wrapper",
"sync_wrapper 0.1.2",
"tower",
"tower-layer",
"tower-service",
@ -176,8 +182,8 @@ dependencies = [
"async-trait",
"bytes",
"futures-util",
"http",
"http-body",
"http 0.2.12",
"http-body 0.4.6",
"mime",
"rustversion",
"tower-layer",
@ -205,6 +211,12 @@ version = "0.21.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
[[package]]
name = "base64"
version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "bitflags"
version = "1.3.2"
@ -742,7 +754,26 @@ dependencies = [
"futures-core",
"futures-sink",
"futures-util",
"http",
"http 0.2.12",
"indexmap 2.5.0",
"slab",
"tokio",
"tokio-util",
"tracing",
]
[[package]]
name = "h2"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205"
dependencies = [
"atomic-waker",
"bytes",
"fnv",
"futures-core",
"futures-sink",
"http 1.1.0",
"indexmap 2.5.0",
"slab",
"tokio",
@ -768,7 +799,7 @@ version = "7.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d"
dependencies = [
"base64",
"base64 0.21.7",
"byteorder",
"flate2",
"nom",
@ -812,6 +843,17 @@ dependencies = [
"itoa",
]
[[package]]
name = "http"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258"
dependencies = [
"bytes",
"fnv",
"itoa",
]
[[package]]
name = "http-body"
version = "0.4.6"
@ -819,7 +861,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2"
dependencies = [
"bytes",
"http",
"http 0.2.12",
"pin-project-lite",
]
[[package]]
name = "http-body"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
dependencies = [
"bytes",
"http 1.1.0",
]
[[package]]
name = "http-body-util"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f"
dependencies = [
"bytes",
"futures-util",
"http 1.1.0",
"http-body 1.0.1",
"pin-project-lite",
]
@ -851,9 +916,9 @@ dependencies = [
"futures-channel",
"futures-core",
"futures-util",
"h2",
"http",
"http-body",
"h2 0.3.26",
"http 0.2.12",
"http-body 0.4.6",
"httparse",
"httpdate",
"itoa",
@ -865,13 +930,50 @@ dependencies = [
"want",
]
[[package]]
name = "hyper"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a"
dependencies = [
"bytes",
"futures-channel",
"futures-util",
"h2 0.4.6",
"http 1.1.0",
"http-body 1.0.1",
"httparse",
"itoa",
"pin-project-lite",
"smallvec",
"tokio",
"want",
]
[[package]]
name = "hyper-rustls"
version = "0.27.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333"
dependencies = [
"futures-util",
"http 1.1.0",
"hyper 1.5.0",
"hyper-util",
"rustls",
"rustls-pki-types",
"tokio",
"tokio-rustls",
"tower-service",
]
[[package]]
name = "hyper-timeout"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1"
dependencies = [
"hyper",
"hyper 0.14.30",
"pin-project-lite",
"tokio",
"tokio-io-timeout",
@ -879,15 +981,37 @@ dependencies = [
[[package]]
name = "hyper-tls"
version = "0.5.0"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0"
dependencies = [
"bytes",
"hyper",
"http-body-util",
"hyper 1.5.0",
"hyper-util",
"native-tls",
"tokio",
"tokio-native-tls",
"tower-service",
]
[[package]]
name = "hyper-util"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b"
dependencies = [
"bytes",
"futures-channel",
"futures-util",
"http 1.1.0",
"http-body 1.0.1",
"hyper 1.5.0",
"pin-project-lite",
"socket2",
"tokio",
"tower-service",
"tracing",
]
[[package]]
@ -1559,20 +1683,24 @@ checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
[[package]]
name = "reqwest"
version = "0.11.27"
version = "0.12.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62"
checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b"
dependencies = [
"base64",
"base64 0.22.1",
"bytes",
"encoding_rs",
"futures-channel",
"futures-core",
"futures-util",
"h2",
"http",
"http-body",
"hyper",
"h2 0.4.6",
"http 1.1.0",
"http-body 1.0.1",
"http-body-util",
"hyper 1.5.0",
"hyper-rustls",
"hyper-tls",
"hyper-util",
"ipnet",
"js-sys",
"log",
@ -1585,7 +1713,7 @@ dependencies = [
"serde",
"serde_json",
"serde_urlencoded",
"sync_wrapper",
"sync_wrapper 1.0.1",
"system-configuration",
"tokio",
"tokio-native-tls",
@ -1594,7 +1722,22 @@ dependencies = [
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
"winreg",
"windows-registry",
]
[[package]]
name = "ring"
version = "0.17.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
dependencies = [
"cc",
"cfg-if",
"getrandom",
"libc",
"spin",
"untrusted",
"windows-sys 0.52.0",
]
[[package]]
@ -1623,12 +1766,42 @@ dependencies = [
]
[[package]]
name = "rustls-pemfile"
version = "1.0.4"
name = "rustls"
version = "0.23.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c"
checksum = "5fbb44d7acc4e873d613422379f69f237a1b141928c02f6bc6ccfddddc2d7993"
dependencies = [
"base64",
"once_cell",
"rustls-pki-types",
"rustls-webpki",
"subtle",
"zeroize",
]
[[package]]
name = "rustls-pemfile"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50"
dependencies = [
"rustls-pki-types",
]
[[package]]
name = "rustls-pki-types"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b"
[[package]]
name = "rustls-webpki"
version = "0.102.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9"
dependencies = [
"ring",
"rustls-pki-types",
"untrusted",
]
[[package]]
@ -1824,6 +1997,12 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "spin"
version = "0.9.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
[[package]]
name = "stable_deref_trait"
version = "1.2.0"
@ -1862,6 +2041,12 @@ version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]]
name = "subtle"
version = "2.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "syn"
version = "1.0.109"
@ -1891,21 +2076,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
[[package]]
name = "system-configuration"
version = "0.5.1"
name = "sync_wrapper"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7"
checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394"
dependencies = [
"bitflags 1.3.2",
"futures-core",
]
[[package]]
name = "system-configuration"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b"
dependencies = [
"bitflags 2.6.0",
"core-foundation",
"system-configuration-sys",
]
[[package]]
name = "system-configuration-sys"
version = "0.5.0"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9"
checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4"
dependencies = [
"core-foundation-sys",
"libc",
@ -2038,6 +2232,17 @@ dependencies = [
"tokio",
]
[[package]]
name = "tokio-rustls"
version = "0.26.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4"
dependencies = [
"rustls",
"rustls-pki-types",
"tokio",
]
[[package]]
name = "tokio-stream"
version = "0.1.15"
@ -2071,12 +2276,12 @@ dependencies = [
"async-stream",
"async-trait",
"axum",
"base64",
"base64 0.21.7",
"bytes",
"h2",
"http",
"http-body",
"hyper",
"h2 0.3.26",
"http 0.2.12",
"http-body 0.4.6",
"hyper 0.14.30",
"hyper-timeout",
"percent-encoding",
"pin-project",
@ -2201,6 +2406,12 @@ version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d"
[[package]]
name = "untrusted"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]]
name = "url"
version = "2.5.2"
@ -2340,7 +2551,37 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
dependencies = [
"windows-sys 0.52.0",
"windows-sys 0.59.0",
]
[[package]]
name = "windows-registry"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0"
dependencies = [
"windows-result",
"windows-strings",
"windows-targets 0.52.6",
]
[[package]]
name = "windows-result"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e"
dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "windows-strings"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"
dependencies = [
"windows-result",
"windows-targets 0.52.6",
]
[[package]]
@ -2491,16 +2732,6 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "winreg"
version = "0.50.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1"
dependencies = [
"cfg-if",
"windows-sys 0.48.0",
]
[[package]]
name = "workspace-hack"
version = "0.1.0"
@ -2533,3 +2764,9 @@ dependencies = [
"quote",
"syn 2.0.77",
]
[[package]]
name = "zeroize"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"

View file

@ -27,7 +27,7 @@ log = "0.4.21"
openssl = { version = "0.10.63" }
pretty_env_logger = "0.5.0"
rayon = "1.9.0"
reqwest = { version = "0.11.24", features = ["blocking"] }
reqwest = { version = "0.12", features = ["blocking"] }
scraper = "0.18.1"
serde = "1.0.197"
serde_json = "1.0.114"

View file

@ -1,9 +1,10 @@
// SPDX-FileCopyrightText: 2023-2024 Christina Sørensen
// SPDX-FileContributor: Christina Sørensen
// SPDX-FileContributor: Maximilian Marx
//
// SPDX-License-Identifier: EUPL-1.2
use clap::{arg, command, crate_authors, value_parser, ArgAction, Command};
use clap::{arg, command, crate_authors, value_parser, ArgAction, ArgGroup, Command};
const DEFAULT_CACHE: &str = "cache.nixos.org";
@ -38,4 +39,11 @@ pub fn build_cli() -> Command {
.long("print-build-logs")
.conflicts_with("verbose"),
)
.arg(arg!(-'4' --"only-ipv4" "Use IPv4 addresses only.").action(ArgAction::SetTrue))
.arg(arg!(-'6' --"only-ipv6" "Use IPv6 addresses only.").action(ArgAction::SetTrue))
.group(
ArgGroup::new("address_family")
.args(["only-ipv4", "only-ipv6"])
.required(false),
)
}

View file

@ -1,15 +1,17 @@
// SPDX-FileCopyrightText: 2024 Christina Sørensen
// SPDX-FileContributor: Christina Sørensen
// SPDX-FileContributor: Maximilian Marx
//
// SPDX-License-Identifier: EUPL-1.2
use std::sync::Arc;
use std::time::Instant;
use std::{env, io, net::SocketAddr};
use dns_lookup::lookup_host;
use futures::future::join_all;
use gethostname::gethostname;
use itertools::Itertools;
use net::AddressFamilyFilter;
use crate::nix::get_requisites;
@ -91,14 +93,26 @@ async fn main() -> io::Result<()> {
config_dir = DEFAULT_CONFIG_DIR.to_string();
}
let address_family_filter = if matches.get_flag("only-ipv4") {
AddressFamilyFilter::OnlyIPv4
} else if matches.get_flag("only-ipv6") {
AddressFamilyFilter::OnlyIPv6
} else {
Default::default()
};
let domain = cache_url.to_owned();
let ips: Vec<std::net::IpAddr> = lookup_host(&domain).unwrap();
let ips: Vec<std::net::IpAddr> = address_family_filter
.lookup_host(&domain)
.unwrap()
.collect();
log::debug!("{:#?}", &ips);
let domain_addr = SocketAddr::new(ips[0], 443);
let client = reqwest::Client::builder()
.dns_resolver(Arc::new(address_family_filter))
.resolve(&domain, domain_addr)
.build()
.unwrap();

View file

@ -1,15 +1,59 @@
// SPDX-FileCopyrightText: 2024 Christina Sørensen
// SPDX-FileContributor: Christina Sørensen
// SPDX-FileContributor: Maximilian Marx
//
// SPDX-License-Identifier: EUPL-1.2
use std::time::Duration;
use std::{
io,
net::{IpAddr, SocketAddr},
time::Duration,
};
use reqwest::Client;
use reqwest::{dns::Resolve, Client};
use tokio::time::sleep;
const MAX_SLIDE: u64 = 1000;
#[derive(Debug, Copy, Clone, Default)]
pub enum AddressFamilyFilter {
#[default]
Both,
OnlyIPv4,
OnlyIPv6,
}
impl AddressFamilyFilter {
pub fn lookup_host(self, host: &str) -> io::Result<impl Iterator<Item = IpAddr>> {
let addresses = dns_lookup::lookup_host(host)?;
Ok(self.filter_addresses(addresses))
}
fn filter_addresses<T>(self, addresses: T) -> impl Iterator<Item = IpAddr>
where
T: IntoIterator<Item = IpAddr>,
{
addresses.into_iter().filter(move |address| match self {
Self::Both => true,
Self::OnlyIPv4 => matches!(address, IpAddr::V4(_)),
Self::OnlyIPv6 => matches!(address, IpAddr::V6(_)),
})
}
}
impl Resolve for AddressFamilyFilter {
fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving {
let filter = *self;
Box::pin(async move {
let addresses = filter.lookup_host(name.as_str())?;
let socket_addresses: Box<dyn Iterator<Item = SocketAddr> + Send> =
Box::new(addresses.map(|ip| SocketAddr::new(ip, 0)));
Ok(socket_addresses)
})
}
}
pub async fn nar_exists(client: Client, domain: &str, hash: &str, slide: u64) -> usize {
let response = client
.head(format!("https://{domain}/{hash}.narinfo"))

View file

@ -1,6 +1,7 @@
# SPDX-FileCopyrightText: 2024 Christina Sørensen
#
# SPDX-License-Identifier: EUPL-1.2
# SPDX-FileContributor: Maximilian Marx
# This file is generated by `cargo hakari`.
# To regenerate, run:
@ -12,6 +13,7 @@ version = "0.1.0"
description = "workspace-hack package, managed by hakari"
# You can choose to publish this crate: see https://docs.rs/cargo-hakari/latest/cargo_hakari/publishing.
publish = false
edition.workspace = true
# The parts of the file between the BEGIN HAKARI SECTION and END HAKARI SECTION comments
# are managed by hakari.