Merge staging-next into staging

2021-06-28 12:05:23 +00:00 · 2021-06-28 12:05:23 +00:00 · 68af28a926
commit 68af28a926
parent b965dee623 e6001bb168
13 changed files with 235 additions and 107 deletions
--- a/doc/doc-support/lib-function-docs.nix
+++ b/doc/doc-support/lib-function-docs.nix
@ -22,5 +22,6 @@ with pkgs; stdenv.mkDerivation {
    docgen lists 'List manipulation functions'
    docgen debug 'Debugging functions'
    docgen options 'NixOS / nixpkgs option handling'
+    docgen sources 'Source filtering functions'
  '';
 }
--- a/doc/functions/library.xml
+++ b/doc/functions/library.xml
@ -25,4 +25,6 @@
 <xi:include href="./library/generated/debug.xml" />

 <xi:include href="./library/generated/options.xml" />
+
+ <xi:include href="./library/generated/sources.xml" />
 </section>
--- a/lib/sources.nix
+++ b/lib/sources.nix
@ -1,6 +1,7 @@
 # Functions for copying sources to the Nix store.
 { lib }:

+# Tested in lib/tests/sources.sh
 let
  inherit (builtins)
    hasContext
@ -11,14 +12,13 @@ let
    tryEval
    ;
  inherit (lib)
+    boolToString
    filter
    getAttr
    isString
    pathExists
    readFile
    ;
-in
-rec {

  # Returns the type of a path: regular (for file), symlink, or directory
  pathType = p: getAttr (baseNameOf p) (readDir (dirOf p));
@ -84,18 +84,36 @@ rec {
  #
  cleanSourceWith = { filter ? _path: _type: true, src, name ? null }:
    let
-      isFiltered = src ? _isLibCleanSourceWith;
-      origSrc = if isFiltered then src.origSrc else src;
-      filter' = if isFiltered then name: type: filter name type && src.filter name type else filter;
-      name' = if name != null then name else if isFiltered then src.name else "source";
-    in {
-      inherit origSrc;
-      filter = filter';
-      outPath = builtins.path { filter = filter'; path = origSrc; name = name'; };
-      _isLibCleanSourceWith = true;
-      name = name';
+      orig = toSourceAttributes src;
+    in fromSourceAttributes {
+      inherit (orig) origSrc;
+      filter = path: type: filter path type && orig.filter path type;
+      name = if name != null then name else orig.name;
    };

+  /*
+    Add logging to a source, for troubleshooting the filtering behavior.
+    Type:
+      sources.trace :: sourceLike -> Source
+  */
+  trace =
+    # Source to debug. The returned source will behave like this source, but also log its filter invocations.
+    src:
+    let
+      attrs = toSourceAttributes src;
+    in
+      fromSourceAttributes (
+        attrs // {
+          filter = path: type:
+            let
+              r = attrs.filter path type;
+            in
+              builtins.trace "${attrs.name}.filter ${path} = ${boolToString r}" r;
+        }
+      ) // {
+        satisfiesSubpathInvariant = src ? satisfiesSubpathInvariant && src.satisfiesSubpathInvariant;
+      };
+
  # Filter sources by a list of regular expressions.
  #
  # E.g. `src = sourceByRegex ./my-subproject [".*\.py$" "^database.sql$"]`
@ -110,14 +128,26 @@ rec {
      inherit src;
    };

-  # Get all files ending with the specified suffices from the given
-  # directory or its descendants.  E.g. `sourceFilesBySuffices ./dir
-  # [".xml" ".c"]'.
-  sourceFilesBySuffices = path: exts:
+  /*
+    Get all files ending with the specified suffices from the given
+    source directory or its descendants, omitting files that do not match
+    any suffix. The result of the example below will include files like
+    `./dir/module.c` and `./dir/subdir/doc.xml` if present.
+
+    Type: sourceLike -> [String] -> Source
+
+    Example:
+      sourceFilesBySuffices ./. [ ".xml" ".c" ]
+  */
+  sourceFilesBySuffices =
+    # Path or source containing the files to be returned
+    src:
+    # A list of file suffix strings
+    exts:
    let filter = name: type:
      let base = baseNameOf (toString name);
      in type == "directory" || lib.any (ext: lib.hasSuffix ext base) exts;
-    in cleanSourceWith { inherit filter; src = path; };
+    in cleanSourceWith { inherit filter src; };

  pathIsGitRepo = path: (tryEval (commitIdFromGitRepo path)).success;

@ -177,4 +207,57 @@ rec {
  pathHasContext = builtins.hasContext or (lib.hasPrefix storeDir);

  canCleanSource = src: src ? _isLibCleanSourceWith || !(pathHasContext (toString src));
+
+  # -------------------------------------------------------------------------- #
+  # Internal functions
+  #
+
+  # toSourceAttributes : sourceLike -> SourceAttrs
+  #
+  # Convert any source-like object into a simple, singular representation.
+  # We don't expose this representation in order to avoid having a fifth path-
+  # like class of objects in the wild.
+  # (Existing ones being: paths, strings, sources and x//{outPath})
+  # So instead of exposing internals, we build a library of combinator functions.
+  toSourceAttributes = src:
+    let
+      isFiltered = src ? _isLibCleanSourceWith;
+    in
+    {
+      # The original path
+      origSrc = if isFiltered then src.origSrc else src;
+      filter = if isFiltered then src.filter else _: _: true;
+      name = if isFiltered then src.name else "source";
+    };
+
+  # fromSourceAttributes : SourceAttrs -> Source
+  #
+  # Inverse of toSourceAttributes for Source objects.
+  fromSourceAttributes = { origSrc, filter, name }:
+    {
+      _isLibCleanSourceWith = true;
+      inherit origSrc filter name;
+      outPath = builtins.path { inherit filter name; path = origSrc; };
+    };
+
+in {
+  inherit
+    pathType
+    pathIsDirectory
+    pathIsRegularFile
+
+    pathIsGitRepo
+    commitIdFromGitRepo
+
+    cleanSource
+    cleanSourceWith
+    cleanSourceFilter
+    pathHasContext
+    canCleanSource
+
+    sourceByRegex
+    sourceFilesBySuffices
+
+    trace
+    ;
 }
--- a/lib/tests/release.nix
+++ b/lib/tests/release.nix
@ -26,7 +26,11 @@ pkgs.runCommandNoCC "nixpkgs-lib-tests" {
    nix-store --init

    cp -r ${../.} lib
+    echo "Running lib/tests/modules.sh"
    bash lib/tests/modules.sh

+    echo "Running lib/tests/sources.sh"
+    TEST_LIB=$PWD/lib bash lib/tests/sources.sh
+
    touch $out
 ''
--- a/lib/tests/sources.sh
+++ b/lib/tests/sources.sh
@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Use
+#     || die
+die() {
+  echo >&2 "test case failed: " "$@"
+  exit 1
+}
+
+if test -n "${TEST_LIB:-}"; then
+  export NIX_PATH=nixpkgs="$(dirname "$TEST_LIB")"
+else
+  export NIX_PATH=nixpkgs="$(cd $(dirname ${BASH_SOURCE[0]})/../..; pwd)"
+fi
+
+work="$(mktemp -d)"
+clean_up() {
+  rm -rf "$work"
+}
+trap clean_up EXIT
+cd $work
+
+touch {README.md,module.o,foo.bar}
+
+# nix-instantiate doesn't write out the source, only computing the hash, so
+# this uses the experimental nix command instead.
+
+dir="$(nix eval --raw '(with import <nixpkgs/lib>; "${
+  cleanSource ./.
+}")')"
+(cd $dir; find) | sort -f | diff -U10 - <(cat <<EOF
+.
+./foo.bar
+./README.md
+EOF
+) || die "cleanSource 1"
+
+
+dir="$(nix eval --raw '(with import <nixpkgs/lib>; "${
+  cleanSourceWith { src = '"$work"'; filter = path: type: ! hasSuffix ".bar" path; }
+}")')"
+(cd $dir; find) | sort -f | diff -U10 - <(cat <<EOF
+.
+./module.o
+./README.md
+EOF
+) || die "cleanSourceWith 1"
+
+dir="$(nix eval --raw '(with import <nixpkgs/lib>; "${
+  cleanSourceWith { src = cleanSource '"$work"'; filter = path: type: ! hasSuffix ".bar" path; }
+}")')"
+(cd $dir; find) | sort -f | diff -U10 - <(cat <<EOF
+.
+./README.md
+EOF
+) || die "cleanSourceWith + cleanSource"
+
+echo >&2 tests ok
--- a/nixos/modules/programs/appgate-sdp.nix
+++ b/nixos/modules/programs/appgate-sdp.nix
@ -5,8 +5,7 @@ with lib;
 {
  options = {
    programs.appgate-sdp = {
-      enable = mkEnableOption
-        "AppGate SDP VPN client";
+      enable = mkEnableOption "AppGate SDP VPN client";
    };
  };

@ -17,7 +16,10 @@ with lib;
    systemd = {
      packages = [ pkgs.appgate-sdp ];
      # https://github.com/NixOS/nixpkgs/issues/81138
-      services.appgatedriver.wantedBy =  [ "multi-user.target" ];
+      services.appgatedriver.wantedBy = [ "multi-user.target" ];
+      services.appgate-dumb-resolver.path = [ pkgs.e2fsprogs ];
+      services.appgate-resolver.path = [ pkgs.procps pkgs.e2fsprogs ];
+      services.appgatedriver.path = [ pkgs.e2fsprogs ];
    };
  };
 }
--- a/pkgs/applications/audio/lsp-plugins/default.nix
+++ b/pkgs/applications/audio/lsp-plugins/default.nix
@ -5,13 +5,13 @@

 stdenv.mkDerivation rec {
  pname = "lsp-plugins";
-  version = "1.1.26";
+  version = "1.1.30";

  src = fetchFromGitHub {
    owner = "sadko4u";
    repo = pname;
-    rev = "${pname}-${version}";
-    sha256 = "1apw8zh3a3il4smkjji6bih4vbsymj0hjs10fgkrd4nazqkjvgyd";
+    rev = version;
+    sha256 = "0g0nx05dyjwz2149v3pj6sa9divr26jyqvg2kk1qk48s2n4najkz";
  };

  nativeBuildInputs = [ pkg-config php makeWrapper ];
@ -36,6 +36,8 @@ stdenv.mkDerivation rec {

  buildFlags = [ "release" ];

+  enableParallelBuilding = true;
+
  meta = with lib;
    { description = "Collection of open-source audio plugins";
      longDescription = ''
@ -84,6 +86,8 @@ stdenv.mkDerivation rec {
        - Compressor MidSide - Kompressor MidSide
        - Compressor Mono - Kompressor Mono
        - Compressor Stereo - Kompressor Stereo
+        - Artistic Delay Mono - Künstlerische Verzögerung
+        - Artistic Delay Stereo - Künstlerische Verzögerung
        - Latency Meter - Latenzmessgerät
        - Loudness Compensator Mono - Lautstärke Kompensator Mono
        - Loudness Compensator Stereo - Lautstärke Kompensator Stereo
@ -99,6 +103,9 @@ stdenv.mkDerivation rec {
        - Multiband Compressor MidSide x8 - Multi-band Kompressor MidSide x8
        - Multiband Compressor Mono x8 - Multi-band Kompressor Mono x8
        - Multiband Compressor Stereo x8 - Multi-band Kompressor Stereo x8
+        - Oscilloscope x1 - Oscilloscope x1
+        - Oscilloscope x2 - Oscilloscope x2
+        - Oscilloscope x4 - Oscilloscope x4
        - Oscillator Mono - Oszillator Mono
        - Parametric Equalizer x16 LeftRight - Parametrischer Entzerrer x16 LeftRight
        - Parametric Equalizer x16 MidSide - Parametrischer Entzerrer x16 MidSide
--- a/pkgs/applications/editors/neovim/neovide/default.nix
+++ b/pkgs/applications/editors/neovim/neovide/default.nix
@ -16,6 +16,7 @@
 , makeFontsConf
 , libglvnd
 , libxkbcommon
+, wayland
 , xorg
 }:
 rustPlatform.buildRustPackage rec {
@ -86,7 +87,7 @@ rustPlatform.buildRustPackage rec {

  postFixup = ''
      wrapProgram $out/bin/neovide \
-        --prefix LD_LIBRARY_PATH : ${lib.makeLibraryPath [ libglvnd libxkbcommon xorg.libXcursor xorg.libXext xorg.libXrandr xorg.libXi ]}
+        --prefix LD_LIBRARY_PATH : ${lib.makeLibraryPath [ libglvnd libxkbcommon wayland xorg.libXcursor xorg.libXext xorg.libXrandr xorg.libXi ]}
    '';

  postInstall = ''
--- a/pkgs/applications/networking/appgate-sdp/default.nix
+++ b/pkgs/applications/networking/appgate-sdp/default.nix
@ -2,15 +2,13 @@
 , at-spi2-atk
 , at-spi2-core
 , atk
-, bash
+, autoPatchelfHook
 , cairo
-, coreutils
 , cups
 , curl
 , dbus
 , dnsmasq
 , dpkg
-, e2fsprogs
 , expat
 , fetchurl
 , gdk-pixbuf
@ -20,25 +18,14 @@
 , iproute2
 , krb5
 , lib
-, mesa
 , libdrm
-, libX11
-, libXScrnSaver
-, libXcomposite
-, libXcursor
-, libXdamage
-, libXext
-, libXfixes
-, libXi
-, libXrandr
-, libXrender
-, libXtst
-, libxkbcommon
 , libsecret
 , libuuid
 , libxcb
+, libxkbcommon
 , lttng-ust
 , makeWrapper
+, mesa
 , networkmanager
 , nspr
 , nss
@ -50,6 +37,7 @@
 , stdenv
 , systemd
 , xdg-utils
+, xorg
 , zlib
 }:
 with lib;
@ -69,46 +57,48 @@ let
    gtk3
    icu
    krb5
-    mesa
    libdrm
-    libX11
-    libXScrnSaver
-    libXcomposite
-    libXcursor
-    libXdamage
-    libXext
-    libXfixes
-    libXi
-    libXrandr
-    libXrender
-    libXtst
-    libxkbcommon
    libsecret
    libuuid
    libxcb
+    libxkbcommon
    lttng-ust
+    mesa
    nspr
    nss
    openssl
    pango
    stdenv.cc.cc
    systemd
+    xorg.libX11
+    xorg.libXScrnSaver
+    xorg.libXcomposite
+    xorg.libXcursor
+    xorg.libXdamage
+    xorg.libXext
+    xorg.libXfixes
+    xorg.libXi
+    xorg.libXrandr
+    xorg.libXrender
+    xorg.libXtst
+    xorg.libxkbfile
+    xorg.libxshmfence
    zlib
  ];
-  rpath = lib.makeLibraryPath deps;
 in
 stdenv.mkDerivation rec {
  pname = "appgate-sdp";
  version = "5.4.2";

  src = fetchurl {
-    url = "https://bin.appgate-sdp.com/${lib.versions.majorMinor version}/client/appgate-sdp_${version}_amd64.deb";
+    url = "https://bin.appgate-sdp.com/${versions.majorMinor version}/client/appgate-sdp_${version}_amd64.deb";
    sha256 = "sha256-wAhcTRO/Cd4MG1lfPNDq92yGcu3NOfymucddy92VaXo=";
  };

+  # just patch interpreter
+  autoPatchelfIgnoreMissingDeps = true;
  dontConfigure = true;
  dontBuild = true;
-  enableParallelBuilding = true;

  buildInputs = [
    python37
@ -116,6 +106,7 @@ stdenv.mkDerivation rec {
  ];

  nativeBuildInputs = [
+    autoPatchelfHook
    makeWrapper
    dpkg
  ];
@ -125,62 +116,39 @@ stdenv.mkDerivation rec {
  '';

  installPhase = ''
-    mkdir -p $out/bin
-    ln -s "$out/opt/appgate/appgate" "$out/bin/appgate"
    cp -r $out/usr/share $out/share

-    for file in $out/opt/appgate/linux/appgate-resolver.pre \
-                $out/opt/appgate/linux/appgate-dumb-resolver.pre
-    do
-      substituteInPlace $file \
-        --replace "/bin/sh" "${bash}/bin/sh" \
-        --replace "cat" "${coreutils}/bin/cat" \
-        --replace "chattr" "${e2fsprogs}/bin/chattr" \
-        --replace "mv " "${coreutils}/bin/mv " \
-        --replace "pkill" "${procps}/bin/pkill"
-    done
-
-    for file in $out/lib/systemd/system/appgatedriver.service \
-                $out/lib/systemd/system/appgate-dumb-resolver.service \
-                $out/lib/systemd/system/appgate-resolver.service
-    do
-      substituteInPlace $file \
-        --replace "/bin/sh" "${bash}/bin/sh" \
-        --replace "/opt/" "$out/opt/" \
-        --replace "chattr" "${e2fsprogs}/bin/chattr" \
-        --replace "mv " "${coreutils}/bin/mv "
-    done
+    substituteInPlace $out/lib/systemd/system/appgate-dumb-resolver.service \
+        --replace "/opt/" "$out/opt/"

    substituteInPlace $out/lib/systemd/system/appgatedriver.service \
+        --replace "/opt/" "$out/opt/" \
        --replace "InaccessiblePaths=/mnt /srv /boot /media" "InaccessiblePaths=-/mnt -/srv -/boot -/media"

    substituteInPlace $out/lib/systemd/system/appgate-resolver.service \
+        --replace "/usr/sbin/dnsmasq" "${dnsmasq}/bin/dnsmasq" \
+        --replace "/opt/" "$out/opt/"
+
+    substituteInPlace $out/opt/appgate/linux/nm.py \
        --replace "/usr/sbin/dnsmasq" "${dnsmasq}/bin/dnsmasq"

-    substituteInPlace $out/opt/appgate/linux/nm.py --replace "/usr/sbin/dnsmasq" "${dnsmasq}/bin/dnsmasq"
-    substituteInPlace $out/opt/appgate/linux/set_dns --replace "/etc/appgate.conf" "$out/etc/appgate.conf"
+    substituteInPlace $out/opt/appgate/linux/set_dns \
+        --replace "/etc/appgate.conf" "$out/etc/appgate.conf"

-  '';
+    wrapProgram $out/opt/appgate/service/createdump \
+        --set LD_LIBRARY_PATH "${makeLibraryPath [ stdenv.cc.cc ]}"

-  postFixup = ''
-    find $out -type f -name "*.so" -exec patchelf --set-rpath '$ORIGIN:${rpath}' {} \;
-    for binary in $out/opt/appgate/appgate-driver \
-                  $out/opt/appgate/appgate \
-                  $out/opt/appgate/service/createdump \
-                  $out/opt/appgate/service/appgateservice.bin
-    do
-      patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" --set-rpath "$ORIGIN:$out/opt/appgate/service/:$out/opt/appgate/:${rpath}" $binary
-    done
+    wrapProgram $out/opt/appgate/appgate-driver \
+        --prefix PATH : ${makeBinPath [ iproute2 networkmanager dnsmasq ]} \
+        --set LD_LIBRARY_PATH $out/opt/appgate/service

-    # fail if there are missing dependencies
-    ldd $out/opt/appgate/appgate | grep -i 'not found' && exit 1
-    ldd $out/opt/appgate/service/appgateservice.bin | grep -i 'not found' && exit 1
-    ldd $out/opt/appgate/appgate-driver | grep -i 'not found' && exit 1
+    makeWrapper $out/opt/appgate/Appgate $out/bin/appgate \
+        --prefix PATH : ${makeBinPath [ xdg-utils ]} \
+        --set LD_LIBRARY_PATH $out/opt/appgate:${makeLibraryPath deps}

-    wrapProgram $out/opt/appgate/appgate-driver --prefix PATH : ${lib.makeBinPath [ iproute2 networkmanager dnsmasq ]}
    wrapProgram $out/opt/appgate/linux/set_dns --set PYTHONPATH $PYTHONPATH
-    wrapProgram $out/bin/appgate --prefix PATH : ${lib.makeBinPath [ xdg-utils ]}
  '';
+
  meta = with lib; {
    description = "Appgate SDP (Software Defined Perimeter) desktop client";
    homepage = "https://www.appgate.com/support/software-defined-perimeter-support";
@ -189,3 +157,4 @@ stdenv.mkDerivation rec {
    maintainers = with maintainers; [ ymatsiuk ];
  };
 }
+
--- a/pkgs/development/python-modules/pytibber/default.nix
+++ b/pkgs/development/python-modules/pytibber/default.nix
@ -12,7 +12,7 @@

 buildPythonPackage rec {
  pname = "pytibber";
-  version = "0.17.1";
+  version = "0.18.0";

  disabled = pythonOlder "3.7";

@ -20,7 +20,7 @@ buildPythonPackage rec {
    owner = "Danielhiversen";
    repo = "pyTibber";
    rev = version;
-    sha256 = "1zda9cvg6hy0n7sr2z71lkyl93n1gnzxrvf56lhz13pcsffshhdk";
+    sha256 = "sha256-612BBDgVcdpOsEl2Hc+oCDFmSPGjHvfmVr7i7zdfB/o=";
  };

  propagatedBuildInputs = [
--- a/pkgs/development/tools/ocaml/utop/default.nix
+++ b/pkgs/development/tools/ocaml/utop/default.nix
@ -8,13 +8,13 @@ else

 buildDunePackage rec {
  pname = "utop";
-  version = "2.7.0";
+  version = "2.8.0";

  useDune2 = true;

  src = fetchurl {
    url = "https://github.com/ocaml-community/utop/releases/download/${version}/utop-${version}.tbz";
-    sha256 = "sha256-4GisU98mfDzA8vabvCBEBPA2LMTmRyofxUfjJqY8P90=";
+    sha256 = "0mi571ifjzq4wcjarn8q1b7yl8nxjm1jfx3afac224lqwn6bhb2d";
  };

  nativeBuildInputs = [ makeWrapper ];
--- a/pkgs/servers/web-apps/matomo/default.nix
+++ b/pkgs/servers/web-apps/matomo/default.nix
@ -3,16 +3,16 @@
 let
  versions = {
    matomo = {
-      version = "4.2.1";
-      sha256 = "d3ea7572c5b42f2636da89b9c15dd7ae16da1d06dab0cea2ed93304a960277ac";
+      version = "4.3.1";
+      sha256 = "Ve4P1cVV/uZ59BcQaUZLTTOwpjX7veof9jR0l3Y9xOQ=";
    };

    matomo-beta = {
-      version = "4.2.1";
+      version = "4.3.1";
      # `beta` examples: "b1", "rc1", null
      # TOOD when updating: use null if stable version is >= latest beta or release candidate
      beta = null;
-      sha256 = "d3ea7572c5b42f2636da89b9c15dd7ae16da1d06dab0cea2ed93304a960277ac";
+      sha256 = "Ve4P1cVV/uZ59BcQaUZLTTOwpjX7veof9jR0l3Y9xOQ=";
    };
  };
  common = pname: { version, sha256, beta ? null }:
--- a/pkgs/tools/misc/nix-direnv/default.nix
+++ b/pkgs/tools/misc/nix-direnv/default.nix
@ -12,13 +12,13 @@ let
 in
 stdenv.mkDerivation rec {
  pname = "nix-direnv";
-  version = "1.2.6";
+  version = "1.4.0";

  src = fetchFromGitHub {
    owner = "nix-community";
    repo = "nix-direnv";
    rev = version;
-    sha256 = "sha256-0dCIHgoyNgpxbrPDv26oLdU+npcIgpCQdpX4HzS0vN0=";
+    sha256 = "sha256-BKiuYvxgY2P7GK59jul5l0kHNrJtD2jmsMGmX0+09hY=";
  };

  # Substitute instead of wrapping because the resulting file is