Merge pull request #240419 from lilyinstarlight/feature/prefetch-npm-deps-isahc

prefetch-npm-deps: use isahc instead of ureq

pkgs/build-support/node/fetch-npm-deps/Cargo.toml

@@ -6,15 +6,17 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-anyhow = "1.0.65"
-base64 = "0.13.0"
-digest = "0.10.5"
-rayon = "1.5.3"
-serde = { version = "1.0.145", features = ["derive"] }
-serde_json = "1.0.85"
+anyhow = "1.0.71"
+backoff = "0.4.0"
+base64 = "0.21.2"
+digest = "0.10.7"
+env_logger = "0.10.0"
+isahc = { version = "1.7.2", default_features = false }
+rayon = "1.7.0"
+serde = { version = "1.0.164", features = ["derive"] }
+serde_json = "1.0.99"
 sha1 = "0.10.5"
-sha2 = "0.10.6"
-tempfile = "3.3.0"
-ureq = { version = "2.5.0" }
-url = { version = "2.3.1", features = ["serde"] }
-walkdir = "2.3.2"
+sha2 = "0.10.7"
+tempfile = "3.6.0"
+url = { version = "2.4.0", features = ["serde"] }
+walkdir = "2.3.3"

pkgs/build-support/node/fetch-npm-deps/default.nix

@@ -1,4 +1,4 @@
-{ lib, stdenvNoCC, rustPlatform, makeWrapper, Security, gnutar, gzip, nix, testers, fetchurl, prefetch-npm-deps, fetchNpmDeps }:
+{ lib, stdenvNoCC, rustPlatform, makeWrapper, pkg-config, curl, gnutar, gzip, nix, testers, fetchurl, cacert, prefetch-npm-deps, fetchNpmDeps }:
 
 {
   prefetch-npm-deps = rustPlatform.buildRustPackage {
@@ -16,8 +16,8 @@
 
     cargoLock.lockFile = ./Cargo.lock;
 
-    nativeBuildInputs = [ makeWrapper ];
-    buildInputs = lib.optional stdenvNoCC.isDarwin Security;
+    nativeBuildInputs = [ makeWrapper pkg-config ];
+    buildInputs = [ curl ];
 
     postInstall = ''
       wrapProgram "$out/bin/prefetch-npm-deps" --prefix PATH : ${lib.makeBinPath [ gnutar gzip nix ]}
@@ -165,6 +165,12 @@
 
       dontInstall = true;
 
+      impureEnvVars = lib.fetchers.proxyImpureEnvVars;
+
+      SSL_CERT_FILE = if (hash_.outputHash == "" || hash_.outputHash == lib.fakeSha256 || hash_.outputHash == lib.fakeSha512 || hash_.outputHash == lib.fakeHash)
+        then "${cacert}/etc/ssl/certs/ca-bundle.crt"
+        else "/no-cert-file.crt";
+
       outputHashMode = "recursive";
     } // hash_ // forceGitDeps_);
 }

pkgs/build-support/node/fetch-npm-deps/src/cacache.rs

@@ -1,3 +1,4 @@
+use base64::prelude::{Engine, BASE64_STANDARD};
 use digest::{Digest, Update};
 use serde::{Deserialize, Serialize};
 use sha1::Sha1;
@@ -52,14 +53,14 @@ impl Cache {
         let (algo, hash, integrity) = if let Some(integrity) = integrity {
             let (algo, hash) = integrity.split_once('-').unwrap();
 
-            (algo.to_string(), base64::decode(hash)?, integrity)
+            (algo.to_string(), BASE64_STANDARD.decode(hash)?, integrity)
         } else {
             let hash = Sha512::new().chain(data).finalize();
 
             (
                 String::from("sha512"),
                 hash.to_vec(),
-                format!("sha512-{}", base64::encode(hash)),
+                format!("sha512-{}", BASE64_STANDARD.encode(hash)),
             )
         };
 

pkgs/build-support/node/fetch-npm-deps/src/main.rs

@@ -16,6 +16,7 @@ use walkdir::WalkDir;
 
 mod cacache;
 mod parse;
+mod util;
 
 fn cache_map_path() -> Option<PathBuf> {
     env::var_os("CACHE_MAP_PATH").map(PathBuf::from)
@@ -172,6 +173,8 @@ fn map_cache() -> anyhow::Result<HashMap<Url, String>> {
 }
 
 fn main() -> anyhow::Result<()> {
+    env_logger::init();
+
     let args = env::args().collect::<Vec<_>>();
 
     if args.len() < 2 {
@@ -182,6 +185,18 @@ fn main() -> anyhow::Result<()> {
         process::exit(1);
     }
 
+    if let Ok(jobs) = env::var("NIX_BUILD_CORES") {
+        if !jobs.is_empty() {
+            rayon::ThreadPoolBuilder::new()
+                .num_threads(
+                    jobs.parse()
+                        .expect("NIX_BUILD_CORES must be a whole number"),
+                )
+                .build_global()
+                .unwrap();
+        }
+    }
+
     if args[1] == "--fixup-lockfile" {
         let lock = serde_json::from_str(&fs::read_to_string(&args[2])?)?;
 

pkgs/build-support/node/fetch-npm-deps/src/parse/mod.rs

@@ -3,15 +3,15 @@ use lock::UrlOrString;
 use rayon::prelude::*;
 use serde_json::{Map, Value};
 use std::{
-    fs, io,
+    fs,
+    io::{self, Read},
     process::{Command, Stdio},
-    thread,
-    time::Duration,
 };
 use tempfile::{tempdir, TempDir};
-use ureq::{Error, ErrorKind, Response};
 use url::Url;
 
+use crate::util;
+
 pub mod lock;
 
 pub fn lockfile(content: &str, force_git_deps: bool) -> anyhow::Result<Vec<Package>> {
@@ -106,7 +106,7 @@ impl Package {
 
         let specifics = match get_hosted_git_url(&resolved)? {
             Some(hosted) => {
-                let mut body = get_response(hosted.as_str())?.into_reader();
+                let mut body = util::get_url_with_retry(&hosted)?;
 
                 let workdir = tempdir()?;
 
@@ -157,9 +157,7 @@ impl Package {
             Specifics::Registry { .. } => {
                 let mut body = Vec::new();
 
-                get_response(self.url.as_str())?
-                    .into_reader()
-                    .read_to_end(&mut body)?;
+                util::get_url_with_retry(&self.url)?.read_to_end(&mut body)?;
 
                 Ok(body)
             }
@@ -191,31 +189,6 @@ impl Package {
     }
 }
 
-#[allow(clippy::result_large_err)]
-fn get_response(url: &str) -> Result<Response, Error> {
-    for _ in 0..4 {
-        match ureq::get(url).call() {
-            Err(Error::Status(503 | 429, r)) => {
-                let retry: Option<u64> = r.header("retry-after").and_then(|h| h.parse().ok());
-                let retry = retry.unwrap_or(5);
-                eprintln!("{} for {}, retry in {}", r.status(), r.get_url(), retry);
-                thread::sleep(Duration::from_secs(retry));
-            }
-            Err(Error::Transport(t)) => match t.kind() {
-                ErrorKind::ConnectionFailed | ErrorKind::Dns | ErrorKind::Io => {
-                    let retry = 5;
-                    eprintln!("{} for {}, retry in {}", t.kind(), url, retry);
-                    thread::sleep(Duration::from_secs(retry));
-                }
-                _ => return Err(Error::Transport(t)),
-            },
-            result => return result,
-        };
-    }
-    // Ran out of retries; try one last time and return whatever result we get.
-    ureq::get(url).call()
-}
-
 #[allow(clippy::case_sensitive_file_extension_comparisons)]
 fn get_hosted_git_url(url: &Url) -> anyhow::Result<Option<Url>> {
     if ["git", "git+ssh", "git+https", "ssh"].contains(&url.scheme()) {

pkgs/build-support/node/fetch-npm-deps/src/util.rs

@@ -0,0 +1,45 @@
+use backoff::{retry, ExponentialBackoff};
+use isahc::{
+    config::{CaCertificate, Configurable, RedirectPolicy, SslOption},
+    Body, Request, RequestExt,
+};
+use std::{env, path::Path};
+use url::Url;
+
+pub fn get_url(url: &Url) -> Result<Body, isahc::Error> {
+    let mut request = Request::get(url.as_str()).redirect_policy(RedirectPolicy::Limit(10));
+
+    // Respect SSL_CERT_FILE if environment variable exists
+    if let Ok(ssl_cert_file) = env::var("SSL_CERT_FILE") {
+        if Path::new(&ssl_cert_file).exists() {
+            // When file exists, use it. NIX_SSL_CERT_FILE will still override.
+            request = request.ssl_ca_certificate(CaCertificate::file(ssl_cert_file));
+        } else if env::var("outputHash").is_ok() {
+            // When file does not exist, assume we are downloading in a FOD and
+            // therefore do not need to check certificates, since the output is
+            // already hashed.
+            request = request.ssl_options(SslOption::DANGER_ACCEPT_INVALID_CERTS);
+        }
+    }
+
+    Ok(request.body(())?.send()?.into_body())
+}
+
+pub fn get_url_with_retry(url: &Url) -> Result<Body, isahc::Error> {
+    retry(ExponentialBackoff::default(), || {
+        get_url(url).map_err(|err| {
+            if err.is_network() || err.is_timeout() {
+                backoff::Error::transient(err)
+            } else {
+                backoff::Error::permanent(err)
+            }
+        })
+    })
+    .map_err(|backoff_err| match backoff_err {
+        backoff::Error::Permanent(err)
+        | backoff::Error::Transient {
+            err,
+            retry_after: _,
+        } => err,
+    })
+}

pkgs/top-level/all-packages.nix

@@ -9798,9 +9798,8 @@ with pkgs;
 
   npmHooks = callPackage ../build-support/node/build-npm-package/hooks { };
 
-  inherit (callPackage ../build-support/node/fetch-npm-deps {
-    inherit (darwin.apple_sdk.frameworks) Security;
-  }) fetchNpmDeps prefetch-npm-deps;
+  inherit (callPackage ../build-support/node/fetch-npm-deps { })
+    fetchNpmDeps prefetch-npm-deps;
 
   nodePackages_latest = dontRecurseIntoAttrs nodejs_latest.pkgs;