Merge pull request #154703 from adisbladis/poetry2nix-1_25_0

poetry2nix: 1.24.1 -> 1.26.0

pkgs/development/tools/poetry2nix/poetry2nix/default.nix

@@ -5,7 +5,7 @@
 }:
 let
   # Poetry2nix version
-  version = "1.24.1";
+  version = "1.26.0";
 
   inherit (poetryLib) isCompatible readTOML moduleName;
 
@@ -122,10 +122,10 @@ lib.makeScope pkgs.newScope (self: {
       # Example: { my-app = ./src; }
     , editablePackageSources ? { }
     , __isBootstrap ? false  # Hack: Always add Poetry as a build input unless bootstrapping
+    , pyProject ? readTOML pyproject
     }@attrs:
     let
       poetryPkg = poetry.override { inherit python; };
-      pyProject = readTOML pyproject;
 
       scripts = pyProject.tool.poetry.scripts or { };
       hasScripts = scripts != { };
@@ -133,9 +133,11 @@ lib.makeScope pkgs.newScope (self: {
         inherit python scripts;
       };
 
-      hasEditable = editablePackageSources != { };
+      editablePackageSources' = lib.filterAttrs (name: path: path != null) editablePackageSources;
+      hasEditable = editablePackageSources' != { };
       editablePackage = self.mkPoetryEditablePackage {
-        inherit pyProject python editablePackageSources;
+        inherit pyProject python;
+        editablePackageSources = editablePackageSources';
       };
 
       poetryLock = readTOML poetrylock;
@@ -190,7 +192,10 @@ lib.makeScope pkgs.newScope (self: {
               (lib.reverseList compatible)
           );
         in
-        lockPkgs;
+        lockPkgs // {
+          # Create a dummy null package for the current project in case any dependencies depend on the root project (issue #307)
+          ${pyProject.tool.poetry.name} = null;
+        };
       overlays = builtins.map
         getFunctorFn
         (
@@ -264,14 +269,34 @@ lib.makeScope pkgs.newScope (self: {
     , extraPackages ? ps: [ ]
     }:
     let
+      inherit (lib) elem hasAttr;
+
+      pyProject = readTOML pyproject;
+
+      # Automatically add dependencies with develop = true as editable packages, but only if path dependencies
+      getEditableDeps = set: lib.mapAttrs
+        (name: value: projectDir + "/${value.path}")
+        (lib.filterAttrs (name: dep: dep.develop or false && hasAttr "path" dep) set);
+
+      editablePackageSources' = (
+        (getEditableDeps (pyProject.tool.poetry."dependencies" or { }))
+        // (getEditableDeps (pyProject.tool.poetry."dev-dependencies" or { }))
+        // editablePackageSources
+      );
+
       poetryPython = self.mkPoetryPackages {
-        inherit pyproject poetrylock overrides python pwd preferWheels editablePackageSources;
+        inherit pyproject poetrylock overrides python pwd preferWheels pyProject;
+        editablePackageSources = editablePackageSources';
       };
 
       inherit (poetryPython) poetryPackages;
 
+      # Don't add editable sources to the environment since they will sometimes fail to build and are not useful in the development env
+      editableAttrs = lib.attrNames editablePackageSources';
+      envPkgs = builtins.filter (drv: ! lib.elem (drv.pname or drv.name or "") editableAttrs) poetryPackages;
+
     in
-    poetryPython.python.withPackages (ps: poetryPackages ++ (extraPackages ps));
+    poetryPython.python.withPackages (ps: envPkgs ++ (extraPackages ps));
 
   /* Creates a Python application from pyproject.toml and poetry.lock
 
@@ -282,7 +307,10 @@ lib.makeScope pkgs.newScope (self: {
   */
   mkPoetryApplication =
     { projectDir ? null
-    , src ? self.cleanPythonSources { src = projectDir; }
+    , src ? (
+        # Assume that a project which is the result of a derivation is already adequately filtered
+        if lib.isDerivation projectDir then projectDir else self.cleanPythonSources { src = projectDir; }
+      )
     , pyproject ? projectDir + "/pyproject.toml"
     , poetrylock ? projectDir + "/poetry.lock"
     , overrides ? self.defaultPoetryOverrides

pkgs/development/tools/poetry2nix/poetry2nix/fetch_from_legacy.py

@@ -63,30 +63,36 @@ context.verify_mode = ssl.CERT_NONE
 req = urllib.request.Request(index_url)
 if username and password:
     import base64
-    password_b64 = base64.b64encode(bytes(f"{username}:{password}", "utf-8")).decode("utf-8")
-    req.add_header("Authorization", f"Basic {password_b64}")
-response = urllib.request.urlopen(
-    req,
-    context=context)
+
+    password_b64 = base64.b64encode(":".join((username, password)).encode()).decode(
+        "utf-8"
+    )
+    req.add_header("Authorization", "Basic {}".format(password_b64))
+response = urllib.request.urlopen(req, context=context)
 index = response.read()
 
 parser = Pep503()
 parser.feed(str(index))
 if package_filename not in parser.sources:
-    print("The file %s has not be found in the index %s" % (
-        package_filename, index_url))
+    print(
+        "The file %s has not be found in the index %s" % (package_filename, index_url)
+    )
     exit(1)
 
 package_file = open(package_filename, "wb")
 # Sometimes the href is a relative path
-if urlparse(parser.sources[package_filename]).netloc == '':
+if urlparse(parser.sources[package_filename]).netloc == "":
     parsed_url = urlparse(index_url)
-    package_url = urlunparse((
-        parsed_url.scheme,
-        parsed_url.netloc,
-        parser.sources[package_filename],
-        None, None, None,
-    ))
+    package_url = urlunparse(
+        (
+            parsed_url.scheme,
+            parsed_url.netloc,
+            parsed_url.path + "/" + parser.sources[package_filename],
+            None,
+            None,
+            None,
+        )
+    )
 else:
     package_url = parser.sources[package_filename]
 
@@ -106,10 +112,8 @@ print("Downloading %s" % real_package_url)
 
 req = urllib.request.Request(real_package_url)
 if username and password:
-    req.add_unredirected_header("Authorization", f"Basic {password_b64}")
-response = urllib.request.urlopen(
-    req,
-    context=context)
+    req.add_unredirected_header("Authorization", "Basic {}".format(password_b64))
+response = urllib.request.urlopen(req, context=context)
 
 with response as r:
     shutil.copyfileobj(r, package_file)

pkgs/development/tools/poetry2nix/poetry2nix/hooks/pyproject-without-special-deps.py

@@ -22,7 +22,12 @@ def main(input, output, fields_to_remove):
                 if any_removed:
                     dep["version"] = "*"
 
-    json.dump(data, output, separators=(",", ":"))
+    # Set ensure_ascii to False because TOML is valid UTF-8 so text that can't
+    # be represented in ASCII is perfectly legitimate
+    # HACK: Setting ensure_asscii to False breaks Python2 for some dependencies (like cachy==0.3.0)
+    json.dump(
+        data, output, separators=(",", ":"), ensure_ascii=sys.version_info.major < 3
+    )
 
 
 if __name__ == "__main__":

pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix

@@ -98,12 +98,26 @@ self: super:
     '';
   });
 
+  backports-functools-lru-cache = super.backports-functools-lru-cache.overridePythonAttrs (old: {
+    postPatch = ''
+      substituteInPlace setup.py --replace \
+        'setuptools.setup()' \
+        'setuptools.setup(version="${old.version}")'
+    '';
+  });
+
   bcrypt = super.bcrypt.overridePythonAttrs (
     old: {
       buildInputs = (old.buildInputs or [ ]) ++ [ pkgs.libffi ];
     }
   );
 
+  bjoern = super.bjoern.overridePythonAttrs (
+    old: {
+      buildInputs = (old.nativeBuildInputs or [ ]) ++ [ pkgs.libev ];
+    }
+  );
+
   black = super.black.overridePythonAttrs (
     old: {
       dontPreferSetupPy = true;
@@ -247,6 +261,36 @@ self: super:
     buildInputs = (old.buildInputs or [ ]) ++ [ self.setuptools ];
   });
 
+  dbus-python = super.dbus-python.overridePythonAttrs (old: {
+    outputs = [ "out" "dev" ];
+
+    postPatch = old.postPatch or "" + ''
+      substituteInPlace ./configure --replace /usr/bin/file ${pkgs.file}/bin/file
+      substituteInPlace ./dbus-python.pc.in --replace 'Cflags: -I''${includedir}' 'Cflags: -I''${includedir}/dbus-1.0'
+    '';
+
+    configureFlags = (old.configureFlags or [ ]) ++ [
+      "PYTHON_VERSION=${lib.versions.major self.python.version}"
+    ];
+
+    preConfigure = lib.concatStringsSep "\n" [
+      (old.preConfigure or "")
+      (if (lib.versionAtLeast stdenv.hostPlatform.darwinMinVersion "11" && stdenv.isDarwin) then ''
+        MACOSX_DEPLOYMENT_TARGET=10.16
+      '' else "")
+    ];
+
+    preBuild = old.preBuild or "" + ''
+      make distclean
+    '';
+
+    nativeBuildInputs = old.nativeBuildInputs or [ ] ++ [ pkgs.pkg-config ];
+    buildInputs = old.buildInputs or [ ] ++ [ pkgs.dbus pkgs.dbus-glib ]
+      # My guess why it's sometimes trying to -lncurses.
+      # It seems not to retain the dependency anyway.
+      ++ lib.optional (! self.python ? modules) pkgs.ncurses;
+  });
+
   dcli = super.dcli.overridePythonAttrs (old: {
     propagatedBuildInputs = (old.propagatedBuildInputs or [ ]) ++ [ self.setuptools ];
   });
@@ -373,6 +417,13 @@ self: super:
     }
   );
 
+  fastapi = super.fastapi.overridePythonAttrs (
+    old: {
+      # Note: requires full flit, not just flit-core
+      nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ self.flit ];
+    }
+  );
+
   fastecdsa = super.fastecdsa.overridePythonAttrs (old: {
     buildInputs = old.buildInputs ++ [ pkgs.gmp.dev ];
   });
@@ -504,6 +555,13 @@ self: super:
     propagatedBuildInputs = (old.propagatedBuildInputs or [ ]) ++ [ self.pyparsing ];
   });
 
+  icecream = super.icecream.overridePythonAttrs (old: {
+    #  # ERROR: Could not find a version that satisfies the requirement executing>=0.3.1 (from icecream) (from versions: none)
+    postPatch = ''
+      substituteInPlace setup.py --replace 'executing>=0.3.1' 'executing'
+    '';
+  });
+
   imagecodecs = super.imagecodecs.overridePythonAttrs (
     old: {
       patchPhase = ''
@@ -569,9 +627,9 @@ self: super:
       # disable the removal of pyproject.toml, required because of setuptools_scm
       dontPreferSetupPy = true;
 
-      postPatch = old.postPatch or "" + ''
+      postPatch = old.postPatch or "" + (lib.optionalString ((old.format or "") != "wheel") ''
         substituteInPlace setup.py --replace 'setuptools.setup()' 'setuptools.setup(version="${old.version}")'
-      '';
+      '');
     }
   );
 
@@ -867,13 +925,6 @@ self: super:
     buildInputs = oa.buildInputs ++ [ self.pbr ];
   });
 
-  moto = super.moto.overridePythonAttrs (
-    old: {
-      buildInputs = (old.buildInputs or [ ]) ++
-        [ self.sshpubkeys ];
-    }
-  );
-
   mpi4py = super.mpi4py.overridePythonAttrs (
     old:
     let
@@ -988,8 +1039,18 @@ self: super:
   );
 
   opencv-python = super.opencv-python.overridePythonAttrs (
-    old: rec {
-      buildInputs = (old.buildInputs or [ ]) ++ [ self.scikit-build ];
+    old: {
+      nativeBuildInputs = [ pkgs.cmake ] ++ old.nativeBuildInputs;
+      buildInputs = [ self.scikit-build ] ++ (old.buildInputs or [ ]);
+      dontUseCmakeConfigure = true;
+    }
+  );
+
+  opencv-contrib-python = super.opencv-contrib-python.overridePythonAttrs (
+    old: {
+      nativeBuildInputs = [ pkgs.cmake ] ++ old.nativeBuildInputs;
+      buildInputs = [ self.scikit-build ] ++ (old.buildInputs or [ ]);
+      dontUseCmakeConfigure = true;
     }
   );
 
@@ -1007,6 +1068,13 @@ self: super:
     }
   );
 
+  pantalaimon = super.pantalaimon.overridePythonAttrs (old: {
+    nativeBuildInputs = old.nativeBuildInputs or [ ] ++ [ pkgs.installShellFiles ];
+    postInstall = old.postInstall or "" + ''
+      installManPage docs/man/*.[1-9]
+    '';
+  });
+
   paramiko = super.paramiko.overridePythonAttrs (old: {
     doCheck = false; # requires networking
   });
@@ -1077,6 +1145,10 @@ self: super:
     }
   );
 
+  prettytable = super.prettytable.overridePythonAttrs (old: {
+    propagatedBuildInputs = (old.propagatedBuildInputs or [ ]) ++ [ self.setuptools ];
+  });
+
   psycopg2 = super.psycopg2.overridePythonAttrs (
     old: {
       buildInputs = (old.buildInputs or [ ])
@@ -1277,7 +1349,11 @@ self: super:
     }
   );
 
-  pytezos = super.pytezos.override (old: {
+  pytaglib = super.pytaglib.overridePythonAttrs (old: {
+    buildInputs = (old.buildInputs or [ ]) ++ [ pkgs.taglib ];
+  });
+
+  pytezos = super.pytezos.overridePythonAttrs (old: {
     buildInputs = (old.buildInputs or [ ]) ++ [ pkgs.libsodium ];
   });
 
@@ -1323,6 +1399,7 @@ self: super:
           pkgs.qt5.qtsvg
           pkgs.qt5.qtdeclarative
           pkgs.qt5.qtwebchannel
+          pkgs.qt5.qt3d
           # self.pyqt5-sip
           self.sip
         ]
@@ -1472,6 +1549,12 @@ self: super:
     }
   );
 
+  python-olm = super.python-olm.overridePythonAttrs (
+    old: {
+      buildInputs = old.buildInputs or [ ] ++ [ pkgs.olm ];
+    }
+  );
+
   python-snappy = super.python-snappy.overridePythonAttrs (
     old: {
       buildInputs = (old.buildInputs or [ ]) ++ [ pkgs.snappy ];
@@ -1496,6 +1579,13 @@ self: super:
     }
   );
 
+  pyudev = super.pyudev.overridePythonAttrs (old: {
+    postPatch = ''
+      substituteInPlace src/pyudev/_ctypeslib/utils.py \
+        --replace "find_library(name)" "'${pkgs.lib.getLib pkgs.systemd}/lib/libudev.so'"
+    '';
+  });
+
   pyusb = super.pyusb.overridePythonAttrs (
     old: {
       postPatch = ''
@@ -1559,6 +1649,12 @@ self: super:
     }
   );
 
+  requests-mock = super.requests-mock.overridePythonAttrs (
+    old: {
+      propagatedBuildInputs = (old.propagatedBuildInputs or [ ]) ++ [ super.pbr ];
+    }
+  );
+
   requests-unixsocket = super.requests-unixsocket.overridePythonAttrs (
     old: {
       nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ self.pbr ];
@@ -1616,6 +1712,18 @@ self: super:
     } else old
   );
 
+  scikit-image = super.scikit-image.overridePythonAttrs (
+    old: {
+      nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [
+        self.cython
+        self.pythran
+        self.packaging
+        self.wheel
+        self.numpy
+      ];
+    }
+  );
+
   scikit-learn = super.scikit-learn.overridePythonAttrs (
     old: {
       buildInputs = (old.buildInputs or [ ]) ++ [
@@ -1673,7 +1781,7 @@ self: super:
   tables = super.tables.overridePythonAttrs (
     old: {
       buildInputs = (old.buildInputs or [ ]) ++ [ self.pywavelets ];
-      HDF5_DIR = "${pkgs.hdf5}";
+      HDF5_DIR = lib.getDev pkgs.hdf5;
       nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ pkgs.pkg-config ];
       propagatedBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ pkgs.hdf5 self.numpy self.numexpr ];
     }
@@ -2126,6 +2234,10 @@ self: super:
     buildInputs = (old.buildInputs or [ ]) ++ [ self.pbr ];
   });
 
+  pysqlite = super.pysqlite.overridePythonAttrs (old: {
+    buildInputs = (old.buildInputs or [ ]) ++ [ pkgs.sqlite ];
+  });
+
   selinux = super.selinux.overridePythonAttrs (old: {
     buildInputs = (old.buildInputs or [ ]) ++ [ self.setuptools-scm-git-archive ];
   });
@@ -2147,6 +2259,12 @@ self: super:
     sourceRoot = ".";
   });
 
+  wcwidth = super.wcwidth.overridePythonAttrs (old: {
+    propagatedBuildInputs = (old.propagatedBuildInputs or [ ]) ++
+      lib.optional self.isPy27 (self.backports-functools-lru-cache or self.backports_functools_lru_cache)
+    ;
+  });
+
   wtforms = super.wtforms.overridePythonAttrs (old: {
     buildInputs = (old.buildInputs or [ ]) ++ [ self.Babel ];
   });

pkgs/development/tools/poetry2nix/poetry2nix/pep425.nix

@@ -84,7 +84,13 @@ let
             )
           else
             (x: x.platform == "any")
-        else (x: hasInfix "macosx" x.platform || x.platform == "any");
+        else
+          if stdenv.isDarwin
+          then
+            if stdenv.targetPlatform.isAarch64
+            then (x: x.platform == "any" || (hasInfix "macosx" x.platform && lib.lists.any (e: hasSuffix e x.platform) [ "arm64" "aarch64" ]))
+            else (x: x.platform == "any" || (hasInfix "macosx" x.platform && hasSuffix "x86_64" x.platform))
+          else (x: x.platform == "any");
       filterWheel = x:
         let
           f = toWheelAttrs x.file;
@@ -93,7 +99,7 @@ let
       filtered = builtins.filter filterWheel filesWithoutSources;
       choose = files:
         let
-          osxMatches = [ "10_12" "10_11" "10_10" "10_9" "10_8" "10_7" "any" ];
+          osxMatches = [ "12_0" "11_0" "10_12" "10_11" "10_10" "10_9" "10_8" "10_7" "any" ];
           linuxMatches = [ "manylinux1_" "manylinux2010_" "manylinux2014_" "any" ];
           chooseLinux = x: lib.take 1 (findBestMatches linuxMatches x);
           chooseOSX = x: lib.take 1 (findBestMatches osxMatches x);