Merge pull request #136777 from lukegb/buildcatrust

use buildcatrust to generate trust stores

nixos/modules/security/ca.nix

@@ -8,12 +8,10 @@ let
 
   cacertPackage = pkgs.cacert.override {
     blacklist = cfg.caCertificateBlacklist;
+    extraCertificateFiles = cfg.certificateFiles;
+    extraCertificateStrings = cfg.certificates;
   };
-
+  caBundle = "${cacertPackage}/etc/ssl/certs/ca-bundle.crt";
-  caCertificates = pkgs.runCommand "ca-certificates.crt" {
-    files = cfg.certificateFiles ++ [ (builtins.toFile "extra.crt" (concatStringsSep "\n" cfg.certificates)) ];
-    preferLocalBuild = true;
-  } "awk 1 $files > $out";  # awk ensures a newline between each pair of consecutive files
 
 in
 
@@ -74,16 +72,17 @@ in
 
   config = {
 
-    security.pki.certificateFiles = [ "${cacertPackage}/etc/ssl/certs/ca-bundle.crt" ];
-
     # NixOS canonical location + Debian/Ubuntu/Arch/Gentoo compatibility.
-    environment.etc."ssl/certs/ca-certificates.crt".source = caCertificates;
+    environment.etc."ssl/certs/ca-certificates.crt".source = caBundle;
 
     # Old NixOS compatibility.
-    environment.etc."ssl/certs/ca-bundle.crt".source = caCertificates;
+    environment.etc."ssl/certs/ca-bundle.crt".source = caBundle;
 
     # CentOS/Fedora compatibility.
-    environment.etc."pki/tls/certs/ca-bundle.crt".source = caCertificates;
+    environment.etc."pki/tls/certs/ca-bundle.crt".source = caBundle;
+
+    # P11-Kit trust source.
+    environment.etc."ssl/trust-source".source = "${cacertPackage.p11kit}/etc/ssl/trust-source";
 
   };
 

nixos/tests/custom-ca.nix

@@ -109,9 +109,7 @@ in
 
       environment.systemPackages = with pkgs; [
         xdotool
-        # Firefox was disabled here, because we needed to disable p11-kit support in nss,
+        firefox
-        # which is why it will not use the system certificate store for the time being.
-        # firefox
         chromium
         qutebrowser
         midori
@@ -153,9 +151,7 @@ in
         machine.fail("curl -fv https://bad.example.com")
 
     browsers = {
-      # Firefox was disabled here, because we needed to disable p11-kit support in nss,
+      "firefox": "Security Risk",
-      # which is why it will not use the system certificate store for the time being.
-      #"firefox": "Security Risk",
       "chromium": "not private",
       "qutebrowser -T": "Certificate error",
       "midori": "Security"

pkgs/applications/networking/browsers/firefox/common.nix

@@ -117,9 +117,7 @@ let
                 then overrideCC stdenv llvmPackages.clangUseLLVM
                 else stdenv;
 
-  # Disable p11-kit support in nss until our cacert packages has caught up exposing CKA_NSS_MOZILLA_CA_POLICY
+  nss_pkg = if lib.versionOlder version "83" then nss_3_53 else nss;
-  # https://github.com/NixOS/nixpkgs/issues/126065
-  nss_pkg = if lib.versionOlder version "83" then nss_3_53 else nss.override { useP11kit = false; };
 
   # --enable-release adds -ffunction-sections & LTO that require a big amount of
   # RAM and the 32-bit memory space cannot handle that linking

pkgs/data/misc/cacert/default.nix

@@ -1,126 +1,27 @@
 { lib
 , stdenv
+, writeText
 , fetchurl
-, nss
+, buildcatrust
-, python3
+, blacklist ? []
-, blacklist ? [ ]
+, extraCertificateFiles ? []
+, extraCertificateStrings ? []
 
-  # Used for tests only
+# Used by update.sh
+, nssOverride ? null
+
+# Used for tests only
 , runCommand
 , cacert
 , openssl
 }:
 
 let
-  certdata2pem = fetchurl {
+  blocklist = writeText "cacert-blocklist.txt" (lib.concatStringsSep "\n" blacklist);
-    name = "certdata2pem.py";
+  extraCertificatesBundle = writeText "cacert-extra-certificates-bundle.crt" (lib.concatStringsSep "\n\n" extraCertificateStrings);
-    urls = [
-      "https://salsa.debian.org/debian/ca-certificates/raw/debian/20170717/mozilla/certdata2pem.py"
-      "https://git.launchpad.net/ubuntu/+source/ca-certificates/plain/mozilla/certdata2pem.py?id=47e49e1e0a8a1ca74deda27f88fe181191562957"
-    ];
-    sha256 = "1d4q27j1gss0186a5m8bs5dk786w07ccyq0qi6xmd2zr1a8q16wy";
-  };
-in
-
-stdenv.mkDerivation rec {
-  pname = "nss-cacert";
-  version = "3.71";
-
-  src = fetchurl {
-    url = "mirror://mozilla/security/nss/releases/NSS_${lib.replaceStrings ["."] ["_"] version}_RTM/src/nss-${version}.tar.gz";
-    sha256 = "0ly2l3dv6z5hlxs72h5x6796ni3x1bq60saavaf42ddgv4ax7b4r";
-  };
-
-  outputs = [ "out" "unbundled" ];
-
-  nativeBuildInputs = [ python3 ];
-
-  configurePhase = ''
-    ln -s nss/lib/ckfw/builtins/certdata.txt
-
-    cat << EOF > blacklist.txt
-    ${lib.concatStringsSep "\n" (map (c: ''"${c}"'') blacklist)}
-    EOF
-
-    # copy from the store, otherwise python will scan it for imports
-    cat "${certdata2pem}" > certdata2pem.py
-  '';
-
-  buildPhase = ''
-    python certdata2pem.py | grep -vE '^(!|UNTRUSTED)'
-
-    for cert in *.crt; do
-      echo $cert | cut -d. -f1 | sed -e 's,_, ,g' >> ca-bundle.crt
-      cat $cert >> ca-bundle.crt
-      echo >> ca-bundle.crt
-    done
-  '';
-
-  installPhase = ''
-    mkdir -pv $out/etc/ssl/certs
-    cp -v ca-bundle.crt $out/etc/ssl/certs
-    # install individual certs in unbundled output
-    mkdir -pv $unbundled/etc/ssl/certs
-    cp -v *.crt $unbundled/etc/ssl/certs
-    rm $unbundled/etc/ssl/certs/ca-bundle.crt  # not wanted in unbundled
-  '';
-
-  setupHook = ./setup-hook.sh;
-
-  passthru = {
-    updateScript = ./update.sh;
-    tests = {
-      # Test that building this derivation with a blacklist works, and that UTF-8 is supported.
-      blacklist-utf8 =
-        let
-          blacklistCAToFingerprint = {
-            # "blacklist" uses the CA name from the NSS bundle, but we check for presence using the SHA256 fingerprint.
-            "CFCA EV ROOT" = "5C:C3:D7:8E:4E:1D:5E:45:54:7A:04:E6:87:3E:64:F9:0C:F9:53:6D:1C:CC:2E:F8:00:F3:55:C4:C5:FD:70:FD";
-            "NetLock Arany (Class Gold) Főtanúsítvány" = "6C:61:DA:C3:A2:DE:F0:31:50:6B:E0:36:D2:A6:FE:40:19:94:FB:D1:3D:F9:C8:D4:66:59:92:74:C4:46:EC:98";
-          };
-          mapBlacklist = f: lib.concatStringsSep "\n" (lib.mapAttrsToList f blacklistCAToFingerprint);
-        in
-        runCommand "verify-the-cacert-filter-output"
-          {
-            cacert = cacert.unbundled;
-            cacertWithExcludes = (cacert.override {
-              blacklist = builtins.attrNames blacklistCAToFingerprint;
-            }).unbundled;
-
-            nativeBuildInputs = [ openssl ];
-          } ''
-          isPresent() {
-            # isPresent <unbundled-dir> <ca name> <ca sha256 fingerprint>
-            for f in $1/etc/ssl/certs/*.crt; do
-              fingerprint="$(openssl x509 -in "$f" -noout -fingerprint -sha256 | cut -f2 -d=)"
-              if [[ "x$fingerprint" == "x$3" ]]; then
-                return 0
-              fi
-            done
-            return 1
-          }
-
-          # Ensure that each certificate is in the main "cacert".
-          ${mapBlacklist (caName: caFingerprint: ''
-            isPresent "$cacert" "${caName}" "${caFingerprint}" || ({
-              echo "CA fingerprint ${caFingerprint} (${caName}) is missing from the CA bundle. Consider picking a different CA for the blacklist test." >&2
-              exit 1
-            })
-          '')}
-
-          # Ensure that each certificate is NOT in the "cacertWithExcludes".
-          ${mapBlacklist (caName: caFingerprint: ''
-            isPresent "$cacertWithExcludes" "${caName}" "${caFingerprint}" && ({
-              echo "CA fingerprint ${caFingerprint} (${caName}) is present in the cacertWithExcludes bundle." >&2
-              exit 1
-            })
-          '')}
-
-          touch $out
-        '';
-    };
-  };
 
+  srcVersion = "3.71";
+  version = if nssOverride != null then nssOverride.version else srcVersion;
   meta = with lib; {
     homepage = "https://curl.haxx.se/docs/caextract.html";
     description = "A bundle of X.509 certificates of public Certificate Authorities (CA)";
@@ -128,4 +29,179 @@ stdenv.mkDerivation rec {
     maintainers = with maintainers; [ andir fpletz lukegb ];
     license = licenses.mpl20;
   };
+  certdata = stdenv.mkDerivation {
+    pname = "nss-cacert-certdata";
+    inherit version;
+
+    src = if nssOverride != null then nssOverride.src else fetchurl {
+      url = "mirror://mozilla/security/nss/releases/NSS_${lib.replaceStrings ["."] ["_"] version}_RTM/src/nss-${version}.tar.gz";
+      sha256 = "0ly2l3dv6z5hlxs72h5x6796ni3x1bq60saavaf42ddgv4ax7b4r";
+    };
+
+    dontBuild = true;
+
+    installPhase = ''
+      runHook preInstall
+
+      mkdir $out
+      cp nss/lib/ckfw/builtins/certdata.txt $out
+
+      runHook postInstall
+    '';
+
+    inherit meta;
+  };
+in
+stdenv.mkDerivation rec {
+  pname = "nss-cacert";
+  inherit version;
+
+  src = certdata;
+
+  outputs = [ "out" "unbundled" "p11kit" ];
+
+  nativeBuildInputs = [ buildcatrust ];
+
+  buildPhase = ''
+    mkdir unbundled
+    buildcatrust \
+      --certdata_input certdata.txt \
+      --ca_bundle_input "${extraCertificatesBundle}" ${lib.escapeShellArgs (map (arg: "${arg}") extraCertificateFiles)} \
+      --blocklist "${blocklist}" \
+      --ca_bundle_output ca-bundle.crt \
+      --ca_unpacked_output unbundled \
+      --p11kit_output ca-bundle.trust.p11-kit
+  '';
+
+  installPhase = ''
+    install -D -t "$out/etc/ssl/certs" ca-bundle.crt
+
+    # install p11-kit specific output to p11kit output
+    install -D -t "$p11kit/etc/ssl/trust-source" ca-bundle.trust.p11-kit
+
+    # install individual certs in unbundled output
+    install -D -t "$unbundled/etc/ssl/certs" unbundled/*.crt
+  '';
+
+  setupHook = ./setup-hook.sh;
+
+  passthru = {
+    updateScript = ./update.sh;
+    tests = let
+      isTrusted = ''
+        isTrusted() {
+          # isTrusted <unbundled-dir> <ca name> <ca sha256 fingerprint>
+          for f in $1/etc/ssl/certs/*.crt; do
+            if ! [[ -s "$f" ]]; then continue; fi
+            fingerprint="$(openssl x509 -in "$f" -noout -fingerprint -sha256 | cut -f2 -d=)"
+            if [[ "x$fingerprint" == "x$3" ]]; then
+              # If the certificate is treated as rejected for TLS Web Server, then we consider it untrusted.
+              if openssl x509 -in "$f" -noout -text | grep -q '^Rejected Uses:'; then
+                if openssl x509 -in "$f" -noout -text | grep -A1 '^Rejected Uses:' | grep -q 'TLS Web Server'; then
+                  return 1
+                fi
+              fi
+              return 0
+            fi
+          done
+          return 1
+        }
+      '';
+    in {
+      # Test that building this derivation with a blacklist works, and that UTF-8 is supported.
+      blacklist-utf8 = let
+        blacklistCAToFingerprint = {
+          # "blacklist" uses the CA name from the NSS bundle, but we check for presence using the SHA256 fingerprint.
+          "CFCA EV ROOT" = "5C:C3:D7:8E:4E:1D:5E:45:54:7A:04:E6:87:3E:64:F9:0C:F9:53:6D:1C:CC:2E:F8:00:F3:55:C4:C5:FD:70:FD";
+          "NetLock Arany (Class Gold) Főtanúsítvány" = "6C:61:DA:C3:A2:DE:F0:31:50:6B:E0:36:D2:A6:FE:40:19:94:FB:D1:3D:F9:C8:D4:66:59:92:74:C4:46:EC:98";
+        };
+        mapBlacklist = f: lib.concatStringsSep "\n" (lib.mapAttrsToList f blacklistCAToFingerprint);
+      in runCommand "verify-the-cacert-filter-output" {
+        cacert = cacert.unbundled;
+        cacertWithExcludes = (cacert.override {
+          blacklist = builtins.attrNames blacklistCAToFingerprint;
+        }).unbundled;
+
+        nativeBuildInputs = [ openssl ];
+      } ''
+        ${isTrusted}
+
+        # Ensure that each certificate is in the main "cacert".
+        ${mapBlacklist (caName: caFingerprint: ''
+          isTrusted "$cacert" "${caName}" "${caFingerprint}" || ({
+            echo "CA fingerprint ${caFingerprint} (${caName}) is missing from the CA bundle. Consider picking a different CA for the blacklist test." >&2
+            exit 1
+          })
+        '')}
+
+        # Ensure that each certificate is NOT in the "cacertWithExcludes".
+        ${mapBlacklist (caName: caFingerprint: ''
+          isTrusted "$cacertWithExcludes" "${caName}" "${caFingerprint}" && ({
+            echo "CA fingerprint ${caFingerprint} (${caName}) is present in the cacertWithExcludes bundle." >&2
+            exit 1
+          })
+        '')}
+
+        touch "$out"
+      '';
+
+      # Test that we can add additional certificates to the store, and have them be trusted.
+      extra-certificates = let
+        extraCertificateStr = ''
+          -----BEGIN CERTIFICATE-----
+          MIIB5DCCAWqgAwIBAgIUItvsAYEIdYDkOIo5sdDYMcUaNuIwCgYIKoZIzj0EAwIw
+          KTEnMCUGA1UEAwweTml4T1MgY2FjZXJ0IGV4dHJhIGNlcnRpZmljYXRlMB4XDTIx
+          MDYxMjE5MDQzMFoXDTIyMDYxMjE5MDQzMFowKTEnMCUGA1UEAwweTml4T1MgY2Fj
+          ZXJ0IGV4dHJhIGNlcnRpZmljYXRlMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEuP8y
+          lAm6ZyQt9v/P6gTlV/a9R+D61WjucW04kaegOhg8csiluimYodiSv0Pbgymu+Zxm
+          A3Bz9QGmytaYTiJ16083rJkwwIhqoYl7kWsLzreSTaLz87KH+rdeol59+H0Oo1Mw
+          UTAdBgNVHQ4EFgQUCxuHfvqI4YVU5M+A0+aKvd1LrdswHwYDVR0jBBgwFoAUCxuH
+          fvqI4YVU5M+A0+aKvd1LrdswDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAgNo
+          ADBlAjEArgxgjdNmRlSEuai0dzlktmBEDZKy2Iiul+ttSoce9ohfEVYESwO602HW
+          keVvI56vAjBCro3dc3m2TuktiKO6lQV56PUEyxko4H/sR5pnHlduCGRDlFzQKXf/
+          pMMmtj7cVb8=
+          -----END CERTIFICATE-----
+        '';
+        extraCertificateFile = ./test-cert-file.crt;
+        extraCertificatesToFingerprint = {
+          # String above
+          "NixOS cacert extra certificate string" = "A3:20:D0:84:96:97:25:FF:98:B8:A9:6D:A3:7C:89:95:6E:7A:77:21:92:F3:33:E9:31:AF:5E:03:CE:A9:E5:EE";
+
+          # File
+          "NixOS cacert extra certificate file" = "88:B8:BE:A7:57:AC:F1:FE:D6:98:8B:50:E0:BD:0A:AE:88:C7:DF:70:26:E1:67:5E:F5:F6:91:27:FF:02:D4:A5";
+        };
+        mapExtra = f: lib.concatStringsSep "\n" (lib.mapAttrsToList f extraCertificatesToFingerprint);
+      in runCommand "verify-the-cacert-extra-output" {
+        cacert = cacert.unbundled;
+        cacertWithExtras = (cacert.override {
+          extraCertificateStrings = [ extraCertificateStr ];
+          extraCertificateFiles = [ extraCertificateFile ];
+        }).unbundled;
+
+        nativeBuildInputs = [ openssl ];
+      } ''
+        ${isTrusted}
+
+        # Ensure that the extra certificate is not in the main "cacert".
+        ${mapExtra (extraName: extraFingerprint: ''
+          isTrusted "$cacert" "${extraName}" "${extraFingerprint}" && ({
+            echo "'extra' CA fingerprint ${extraFingerprint} (${extraName}) is present in the main CA bundle." >&2
+            exit 1
+          })
+        '')}
+
+        # Ensure that the extra certificates ARE in the "cacertWithExtras".
+        ${mapExtra (extraName: extraFingerprint: ''
+          isTrusted "$cacertWithExtras" "${extraName}" "${extraFingerprint}" || ({
+            echo "CA fingerprint ${extraFingerprint} (${extraName}) is not present in the cacertWithExtras bundle." >&2
+            exit 1
+          })
+        '')}
+
+        touch "$out"
+      '';
+    };
+  };
+
+  inherit meta;
 }

pkgs/data/misc/cacert/test-cert-file.crt

@@ -0,0 +1,13 @@
+-----BEGIN CERTIFICATE-----
+MIIB7TCCAXSgAwIBAgIUFJB0STXn22fIEDjpncEt++IdFeMwCgYIKoZIzj0EAwIw
+LjEsMCoGA1UEAwwjTml4T1MgY2FjZXJ0IGV4dHJhIGNlcnRpZmljYXRlIGZpbGUw
+HhcNMjEwNjEyMTkxODA4WhcNMjIwNjEyMTkxODA4WjAuMSwwKgYDVQQDDCNOaXhP
+UyBjYWNlcnQgZXh0cmEgY2VydGlmaWNhdGUgZmlsZTB2MBAGByqGSM49AgEGBSuB
+BAAiA2IABMifTLM5K5xd+guGdKE1+NR7wnEJbxw5INzuMrkg/7jgEIQil4+L2YOF
+kU1gxcM80Ot8tQAG5OcSvX1DF6CxunpoCT+hnHqyfqoWFvl89i1BUKjyWCQ5WXEe
+nSkuJUmYC6NTMFEwHQYDVR0OBBYEFBE2kNis1ri4fweyNVRmvje83gFQMB8GA1Ud
+IwQYMBaAFBE2kNis1ri4fweyNVRmvje83gFQMA8GA1UdEwEB/wQFMAMBAf8wCgYI
+KoZIzj0EAwIDZwAwZAIwUZf1qaSb4cezulV+4B4FoJHY2B/nRVIi/rFD8634YEDT
+vcg6dmCi/AqLEzJn7uFMAjBVTu4EVC/mtQCGESFChMeb04fsuhXgttWSwWliVPEG
+jkG7u0UNNGaU8dvrjpqRRmA=
+-----END CERTIFICATE-----

pkgs/data/misc/cacert/update.sh

@@ -28,7 +28,7 @@ BASEDIR="$(dirname "$0")/../../../.."
 
 
 CURRENT_PATH=$(nix-build --no-out-link -A cacert.out)
-PATCHED_PATH=$(nix-build --no-out-link -E "with import $BASEDIR {}; let nss_pkg = pkgs.nss_latest or pkgs.nss; in (cacert.overrideAttrs (_: { inherit (nss_pkg) src version; })).out")
+PATCHED_PATH=$(nix-build --no-out-link -E "with import $BASEDIR {}; let nss_pkg = pkgs.nss_latest or pkgs.nss; in (cacert.override { nssOverride = nss_pkg; }).out")
 
 # Check the hash of the etc subfolder
 # We can't check the entire output as that contains the nix-support folder
@@ -38,5 +38,5 @@ PATCHED_HASH=$(nix-hash "$PATCHED_PATH/etc")
 
 if [[ "$CURRENT_HASH" !=  "$PATCHED_HASH" ]]; then
     NSS_VERSION=$(nix-instantiate --json --eval -E "with import $BASEDIR {}; nss.version" | jq -r .)
-    update-source-version cacert "$NSS_VERSION"
+    update-source-version --version-key=srcVersion cacert.src "$NSS_VERSION"
 fi

pkgs/development/libraries/p11-kit/default.nix

@@ -31,7 +31,7 @@ stdenv.mkDerivation rec {
   configureFlags = [
     "--sysconfdir=/etc"
     "--localstatedir=/var"
-    "--with-trust-paths=/etc/ssl/certs/ca-certificates.crt"
+    "--with-trust-paths=/etc/ssl/trust-source:/etc/ssl/certs/ca-certificates.crt"
   ];
 
   enableParallelBuilding = true;

pkgs/development/python-modules/buildcatrust/default.nix

@@ -0,0 +1,32 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, pytestCheckHook
+}:
+
+buildPythonPackage rec {
+  pname = "buildcatrust";
+  version = "0.1.3";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "sha256:0s0m0fy943dakw9cbd40h46qmrhhgrcp292kppyb34m6y27sbagy";
+  };
+
+  checkInputs = [
+    pytestCheckHook
+  ];
+  disabledTestPaths = [
+    # Non-hermetic, needs internet access (e.g. attempts to retrieve NSS store).
+    "buildcatrust/tests/test_nonhermetic.py"
+  ];
+
+  pythonImportsCheck = [ "buildcatrust" "buildcatrust.cli" ];
+
+  meta = with lib; {
+    description = "Build SSL/TLS trust stores";
+    homepage = "https://github.com/lukegb/buildcatrust";
+    license = licenses.mit;
+    maintainers = with maintainers; [ lukegb ];
+  };
+}

pkgs/top-level/all-packages.nix

@@ -305,6 +305,8 @@ with pkgs;
 
   html5validator = python3Packages.callPackage ../applications/misc/html5validator { };
 
+  buildcatrust = with python3.pkgs; toPythonApplication buildcatrust;
+
   probe-run = callPackage ../development/tools/rust/probe-run {
     inherit (darwin.apple_sdk.frameworks) AppKit IOKit;
   };

pkgs/top-level/python-packages.nix

@@ -1306,6 +1306,8 @@ in {
 
   build = callPackage ../development/python-modules/build { };
 
+  buildcatrust = callPackage ../development/python-modules/buildcatrust { };
+
   bumps = callPackage ../development/python-modules/bumps { };
 
   bunch = callPackage ../development/python-modules/bunch { };