Merge pull request #105477 from adisbladis/poetry2nix-1_14_0

poetry2nix: 1.13.0 -> 1.14.0

pkgs/development/tools/poetry2nix/poetry2nix/bin/poetry2nix

@@ -7,60 +7,127 @@ import toml
 import json
 import sys
 
-
-argparser = argparse.ArgumentParser(description="Poetry2nix CLI")
-
-subparsers = argparser.add_subparsers(dest="subcommand")
-subparsers.required = True
-
-parser_lock = subparsers.add_parser("lock", help="Generate overrides for git hashes",)
-parser_lock.add_argument(
-    "--lock", default="poetry.lock", help="Path to input poetry.lock",
-)
-parser_lock.add_argument(
-    "--out", default="poetry-git-overlay.nix", help="Output file",
-)
+from typing import Dict, Any, Tuple, List
 
 
-def fetch_git(pkg):
-    return (
-        pkg["name"],
-        subprocess.run(
-            [
-                "nix-prefetch-git",
-                "--fetch-submodules",
-                "--url",
-                pkg["source"]["url"],
-                "--rev",
-                pkg["source"]["reference"],
-            ],
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-        ),
+class Package:
+    def __init__(self, attrs: Dict[str, Any]) -> None:
+        self.attrs = attrs
+        self.name = attrs["name"]
+        self.source = self.attrs["source"]
+
+    def fetch(self) -> Tuple["Package", subprocess.CompletedProcess]:
+        raise NotImplementedError()
+
+    def expression(self, output: str) -> str:
+        raise NotImplementedError()
+
+
+class UrlPackage(Package):
+    def fetch(self) -> Tuple[Package, subprocess.CompletedProcess]:
+        return (
+            self,
+            subprocess.run(
+                [
+                    "nix-prefetch-url",
+                    "--unpack",
+                    self.source["url"],
+                ],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True
+            ),
+        )
+
+    def expression(self, output: str) -> str:
+        sha256 = output.rstrip()
+        return textwrap.dedent("""
+            %s = super.%s.overridePythonAttrs (
+              _: {
+                src = pkgs.fetchzip {
+                  url = "%s";
+                  sha256 = "%s";
+                };
+              }
+            );""" % (self.name, self.name, self.source["url"], sha256))
+
+
+class GitPackage(Package):
+    def fetch(self) -> Tuple[Package, subprocess.CompletedProcess]:
+        reference = self.source.get("resolved_reference", self.source["reference"])
+
+        return (
+            self,
+            subprocess.run(
+                [
+                    "nix-prefetch-git",
+                    "--fetch-submodules",
+                    "--url",
+                    self.source["url"],
+                    "--rev",
+                    reference,
+                ],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True
+            ),
+        )
+
+    def expression(self, output: str) -> str:
+        meta = json.loads(output)
+        return textwrap.dedent("""
+            %s = super.%s.overridePythonAttrs (
+              _: {
+                src = pkgs.fetchgit {
+                  url = "%s";
+                  rev = "%s";
+                  sha256 = "%s";
+                };
+              }
+            );""" % (self.name, self.name, meta["url"], meta["rev"], meta["sha256"]))
+
+
+def parse_args() -> argparse.Namespace:
+    argparser = argparse.ArgumentParser(description="Poetry2nix CLI")
+
+    subparsers = argparser.add_subparsers(dest="subcommand")
+    subparsers.required = True
+
+    parser_lock = subparsers.add_parser("lock", help="Generate overrides for git hashes",)
+    parser_lock.add_argument(
+        "--lock", default="poetry.lock", help="Path to input poetry.lock",
     )
+    parser_lock.add_argument(
+        "--out", default="poetry-git-overlay.nix", help="Output file",
+    )
+    return argparser.parse_args()
 
 
-def indent(expr, spaces=2):
+def indent(expr: str, spaces: int = 2) -> str:
     i = " " * spaces
     return "\n".join([(i if l != "" else "") + l for l in expr.split("\n")])
 
 
-if __name__ == "__main__":
-    args = argparser.parse_args()
+def main() -> None:
+    args = parse_args()
 
     with open(args.lock) as lockf:
         lock = toml.load(lockf)
 
-    pkgs = []
+    pkgs: List[Package] = []
     for pkg in lock["package"]:
         if "source" in pkg:
-            pkgs.append(pkg)
+            source_type = pkg["source"]["type"]
+            if source_type == "git":
+                pkgs.append(GitPackage(pkg))
+            elif source_type == "url":
+                pkgs.append(UrlPackage(pkg))
 
     with ThreadPoolExecutor() as e:
         futures = []
 
         for pkg in pkgs:
-            futures.append(e.submit(fetch_git, pkg))
+            futures.append(e.submit(pkg.fetch))
 
         lines = [
             "{ pkgs }:",
@@ -68,30 +135,13 @@ if __name__ == "__main__":
         ]
 
         for f in futures:
-            drv_name, p = f.result()
+            package, p = f.result()
             if p.returncode != 0:
-                sys.stderr.buffer.write(p.stderr)
-                sys.stderr.buffer.flush()
+                sys.stderr.write(p.stderr)
+                sys.stderr.flush()
                 exit(p.returncode)
-
-            meta = json.loads(p.stdout.decode())
-            lines.append(
-                indent(
-                    textwrap.dedent(
-                        """
-              %s = super.%s.overridePythonAttrs (
-                _: {
-                  src = pkgs.fetchgit {
-                    url = "%s";
-                    rev = "%s";
-                    sha256 = "%s";
-                  };
-                }
-              );"""
-                        % (drv_name, drv_name, meta["url"], meta["rev"], meta["sha256"])
-                    )
-                )
-            )
+            expr = package.expression(p.stdout)
+            lines.append(indent(expr))
 
         lines.extend(["", "}", ""])
 
@@ -101,3 +151,7 @@ if __name__ == "__main__":
         fout.write(expr)
 
     print(f"Wrote {args.out}")
+
+
+if __name__ == "__main__":
+    main()

pkgs/development/tools/poetry2nix/poetry2nix/default.nix

@@ -71,7 +71,7 @@ in
 lib.makeScope pkgs.newScope (self: {
 
   # Poetry2nix version
-  version = "1.13.0";
+  version = "1.14.0";
 
   /*
      Returns an attrset { python, poetryPackages, pyProject, poetryLock } for the given pyproject/lockfile.

pkgs/development/tools/poetry2nix/poetry2nix/hooks/default.nix

@@ -1,11 +1,11 @@
 { python
-, callPackage
+, buildPackages
 , makeSetupHook
-, yj
 , wheel
 , pip
 }:
 let
+  callPackage = python.pythonForBuild.pkgs.callPackage;
   pythonInterpreter = python.pythonForBuild.interpreter;
   pythonSitePackages = python.sitePackages;
 in
@@ -20,7 +20,7 @@ in
           deps = [ ];
           substitutions = {
             inherit pythonInterpreter;
-            yj = "${yj}/bin/yj";
+            yj = "${buildPackages.yj}/bin/yj";
             pyprojectPatchScript = "${./pyproject-without-path.py}";
           };
         } ./remove-path-dependencies.sh

pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix

@@ -27,6 +27,7 @@ pythonPackages.callPackage
     , ...
     }@args:
     let
+      inherit (pkgs) stdenv;
       inherit (poetryLib) isCompatible getManyLinuxDeps fetchFromPypi moduleName;
 
       inherit (import ./pep425.nix {
@@ -45,6 +46,7 @@ pythonPackages.callPackage
       toPath = s: pwd + "/${s}";
       isSource = source != null;
       isGit = isSource && source.type == "git";
+      isUrl = isSource && source.type == "url";
       isLocal = isSource && source.type == "directory";
       localDepPath = toPath source.url;
 
@@ -91,7 +93,7 @@ pythonPackages.callPackage
         "toml" # Toml is an extra for setuptools-scm
       ];
       baseBuildInputs = lib.optional (! lib.elem name skipSetupToolsSCM) pythonPackages.setuptools-scm;
-      format = if isLocal then "pyproject" else if isGit then "pyproject" else fileInfo.format;
+      format = if isLocal || isGit || isUrl then "pyproject" else fileInfo.format;
     in
     buildPythonPackage {
       pname = moduleName name;
@@ -113,9 +115,10 @@ pythonPackages.callPackage
 
       buildInputs = (
         baseBuildInputs
+        ++ lib.optional (stdenv.buildPlatform != stdenv.hostPlatform) pythonPackages.setuptools
         ++ lib.optional (!isSource) (getManyLinuxDeps fileInfo.name).pkg
         ++ lib.optional isLocal buildSystemPkgs
-        ++ lib.optional (!__isBootstrap) [ pythonPackages.poetry ]
+        ++ lib.optional (!__isBootstrap) pythonPackages.poetry
       );
 
       propagatedBuildInputs =
@@ -157,14 +160,22 @@ pythonPackages.callPackage
           (
             builtins.fetchGit {
               inherit (source) url;
-              rev = source.reference;
+              rev = source.resolved_reference or source.reference;
               ref = sourceSpec.branch or sourceSpec.rev or sourceSpec.tag or "HEAD";
             }
-          ) else if isLocal then (poetryLib.cleanPythonSources { src = localDepPath; }) else
-        fetchFromPypi {
-          pname = name;
-          inherit (fileInfo) file hash kind;
-        };
+          )
+        else if isUrl then
+          builtins.fetchTarball
+            {
+              inherit (source) url;
+            }
+        else if isLocal then
+          (poetryLib.cleanPythonSources { src = localDepPath; })
+        else
+          fetchFromPypi {
+            pname = name;
+            inherit (fileInfo) file hash kind;
+          };
     }
   )
 { }

pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix

@@ -93,7 +93,7 @@ self: super:
     (
       super.cffi.overridePythonAttrs (
         old: {
-          buildInputs = old.buildInputs ++ [ pkgs.libffi ];
+          buildInputs = old.buildInputs or [ ] ++ [ pkgs.libffi ];
         }
       )
     );
@@ -106,6 +106,12 @@ self: super:
     }
   );
 
+  colour = super.colour.overridePythonAttrs (
+    old: {
+      buildInputs = old.buildInputs ++ [ self.d2to1 ];
+    }
+  );
+
   configparser = super.configparser.overridePythonAttrs (
     old: {
       buildInputs = old.buildInputs ++ [
@@ -120,6 +126,8 @@ self: super:
 
   cryptography = super.cryptography.overridePythonAttrs (
     old: {
+      nativeBuildInputs = old.nativeBuildInputs or [ ]
+        ++ stdenv.lib.optional (stdenv.buildPlatform != stdenv.hostPlatform) self.python.pythonForBuild.pkgs.cffi;
       buildInputs = old.buildInputs ++ [ pkgs.openssl ];
     }
   );
@@ -324,6 +332,17 @@ self: super:
     }
   );
 
+  jira = super.jira.overridePythonAttrs (
+    old: {
+      inherit (pkgs.python3Packages.jira) patches;
+      buildInputs = old.buildInputs ++ [
+        self.pytestrunner
+        self.cryptography
+        self.pyjwt
+      ];
+    }
+  );
+
   jsonpickle = super.jsonpickle.overridePythonAttrs (
     old: {
       dontPreferSetupPy = true;
@@ -499,6 +518,31 @@ self: super:
     buildInputs = oa.buildInputs ++ [ self.pbr ];
   });
 
+  mpi4py = super.mpi4py.overridePythonAttrs (
+    old:
+    let
+      cfg = pkgs.writeTextFile {
+        name = "mpi.cfg";
+        text = (
+          lib.generators.toINI
+            { }
+            {
+              mpi = {
+                mpicc = "${pkgs.openmpi.outPath}/bin/mpicc";
+              };
+            }
+        );
+      };
+    in
+    {
+      propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.openmpi ];
+      enableParallelBuilding = true;
+      preBuild = ''
+        ln -sf ${cfg} mpi.cfg
+      '';
+    }
+  );
+
   multiaddr = super.multiaddr.overridePythonAttrs (
     old: {
       buildInputs = old.buildInputs ++ [ self.pytest-runner ];
@@ -584,8 +628,8 @@ self: super:
       withMysql = old.passthru.withMysql or false;
     in
     {
-      buildInputs = old.buildInputs ++ [ self.cython pkgs.sqlite ];
-      propagatedBuildInputs = old.propagatedBuildInputs
+      buildInputs = old.buildInputs or [ ] ++ [ pkgs.sqlite ];
+      propagatedBuildInputs = old.propagatedBuildInputs or [ ]
         ++ lib.optional withPostgres self.psycopg2
         ++ lib.optional withMysql self.mysql-connector;
     }
@@ -602,8 +646,8 @@ self: super:
     # "Vendor" dependencies (for build-system support)
     postPatch = ''
       echo "import sys" >> poetry/__init__.py
-      for path in ''${PYTHONPATH//:/ }; do echo $path; done | uniq | while read path; do
-        echo "sys.path.insert(0, \"$path\")" >> poetry/__init__.py
+      for path in $propagatedBuildInputs; do
+          echo "sys.path.insert(0, \"$path\")" >> poetry/__init__.py
       done
     '';
 
@@ -796,6 +840,14 @@ self: super:
     }
   );
 
+  python-bugzilla = super.python-bugzilla.overridePythonAttrs (
+    old: {
+      nativeBuildInputs = old.nativeBuildInputs ++ [
+        self.docutils
+      ];
+    }
+  );
+
   python-ldap = super.python-ldap.overridePythonAttrs (
     old: {
       buildInputs = old.buildInputs ++ [ pkgs.openldap pkgs.cyrus_sasl ];
@@ -924,6 +976,15 @@ self: super:
     }
   );
 
+  pytest-django = super.pytest-django.overridePythonAttrs (
+    old: {
+      postPatch = ''
+        substituteInPlace setup.py --replace "'pytest>=3.6'," ""
+        substituteInPlace setup.py --replace "'pytest>=3.6'" ""
+      '';
+    }
+  );
+
   pytest-runner = super.pytest-runner or super.pytestrunner;
 
   python-jose = super.python-jose.overridePythonAttrs (
@@ -1090,6 +1151,43 @@ self: super:
     }
   );
 
+  torch = lib.makeOverridable
+    ({ enableCuda ? false
+     , cudatoolkit ? pkgs.cudatoolkit_10_1
+     , pkg ? super.torch
+     }: pkg.overrideAttrs (old:
+      {
+        preConfigure =
+          if (!enableCuda) then ''
+            export USE_CUDA=0
+          '' else ''
+            export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${cudatoolkit}/targets/x86_64-linux/lib"
+          '';
+        preFixup = lib.optionalString (!enableCuda) ''
+          # For some reason pytorch retains a reference to libcuda even if it
+          # is explicitly disabled with USE_CUDA=0.
+          find $out -name "*.so" -exec ${pkgs.patchelf}/bin/patchelf --remove-needed libcuda.so.1 {} \;
+        '';
+        buildInputs = old.buildInputs ++ lib.optionals enableCuda [
+          pkgs.linuxPackages.nvidia_x11
+          pkgs.nccl.dev
+          pkgs.nccl.out
+        ];
+        propagatedBuildInputs = [
+          super.numpy
+          super.future
+        ];
+      })
+    )
+    { };
+
+  typeguard = super.typeguard.overridePythonAttrs (old: {
+    postPatch = ''
+      substituteInPlace setup.py \
+        --replace 'setup()' 'setup(version="${old.version}")'
+    '';
+  });
+
   # nix uses a dash, poetry uses an underscore
   typing_extensions = super.typing_extensions or self.typing-extensions;
 
@@ -1193,4 +1291,30 @@ self: super:
     }
   );
 
+  credis = super.credis.overridePythonAttrs (
+    old: {
+      buildInputs = old.buildInputs ++ [ self.cython ];
+    }
+  );
+
+  hashids = super.hashids.overridePythonAttrs (
+    old: {
+      buildInputs = old.buildInputs ++ [ self.flit-core ];
+    }
+  );
+
+  supervisor = super.supervisor.overridePythonAttrs (
+    old: {
+      propagatedBuildInputs = old.propagatedBuildInputs ++ [
+        self.meld3
+        self.setuptools
+      ];
+    }
+  );
+
+  cytoolz = super.cytoolz.overridePythonAttrs (
+    old: {
+      propagatedBuildInputs = old.propagatedBuildInputs ++ [ self.toolz ];
+    }
+  );
 }

pkgs/development/tools/poetry2nix/poetry2nix/pep425.nix

@@ -87,7 +87,7 @@ let
       filtered = builtins.filter filterWheel filesWithoutSources;
       choose = files:
         let
-          osxMatches = [ "10_12" "10_11" "10_10" "10_9" "any" ];
+          osxMatches = [ "10_12" "10_11" "10_10" "10_9" "10_8" "10_7" "any" ];
           linuxMatches = [ "manylinux1_" "manylinux2010_" "manylinux2014_" "any" ];
           chooseLinux = x: lib.take 1 (findBestMatches linuxMatches x);
           chooseOSX = x: lib.take 1 (findBestMatches osxMatches x);

pkgs/development/tools/poetry2nix/poetry2nix/pkgs/poetry/default.nix

@@ -15,8 +15,8 @@ poetry2nix.mkPoetryApplication {
   # "Vendor" dependencies (for build-system support)
   postPatch = ''
     echo "import sys" >> poetry/__init__.py
-    for path in ''${PYTHONPATH//:/ }; do echo $path; done | uniq | while read path; do
-      echo "sys.path.insert(0, \"$path\")" >> poetry/__init__.py
+    for path in $propagatedBuildInputs; do
+        echo "sys.path.insert(0, \"$path\")" >> poetry/__init__.py
     done
   '';
 

pkgs/development/tools/poetry2nix/poetry2nix/pkgs/poetry/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "poetry"
-version = "1.1.0"
+version = "1.1.4"
 description = "Python dependency management and packaging made easy."
 authors = [
     "Sébastien Eustace <sebastien@eustace.io>"

pkgs/development/tools/poetry2nix/poetry2nix/pkgs/poetry/src.json

@@ -1,7 +1,7 @@
 {
     "owner": "python-poetry",
     "repo": "poetry",
-    "rev": "539d7f732c34c821258a9853cd3078cbda34a717",
-    "sha256": "0kl23dkq9n112z1pqjg6f1wv3qk77ij6q5glg15lwrj7yrl9k65c",
+    "rev": "8312e3f2dbfa126cd311c666fea30656941e1bd3",
+    "sha256": "0lx3qpz5dad0is7ki5a4vxphvc8cm8fnv4bmrx226a6nvvaj6ahs",
     "fetchSubmodules": true
 }

pkgs/development/tools/poetry2nix/poetry2nix/plugins.nix

@@ -23,6 +23,7 @@ let
 
       dontConfigure = true;
       dontBuild = true;
+      dontUsePythonRecompileBytecode = true;
 
       passthru = {
         inherit (drv.passthru) withPlugins;

pkgs/servers/pinnwand/default.nix

@@ -1,4 +1,4 @@
-{ lib, python3, fetchFromGitHub, poetry, nixosTests }:
+{ lib, python3, fetchFromGitHub, nixosTests }:
 
 let
   python = python3.override {