Merge pull request #111649 from tweag/poetry2nix-1_15_0

poetry2nix: 1.14.0 -> 1.15.2

.github/CODEOWNERS

@@ -76,6 +76,7 @@
 /pkgs/development/interpreters/python   @FRidh
 /pkgs/development/python-modules        @FRidh @jonringer
 /doc/languages-frameworks/python.section.md     @FRidh
+/pkgs/development/tools/poetry2nix @adisbladis
 
 # Haskell
 /pkgs/development/compilers/ghc                       @cdepillabout

pkgs/development/tools/poetry2nix/poetry2nix/default.nix

@@ -1,7 +1,7 @@
 { pkgs ? import <nixpkgs> { }
 , lib ? pkgs.lib
 , poetry ? null
-, poetryLib ? import ./lib.nix { inherit lib pkgs; }
+, poetryLib ? import ./lib.nix { inherit lib pkgs; stdenv = pkgs.stdenv; }
 }:
 let
   inherit (poetryLib) isCompatible readTOML moduleName;
@@ -71,7 +71,40 @@ in
 lib.makeScope pkgs.newScope (self: {
 
   # Poetry2nix version
-  version = "1.14.0";
+  version = "1.15.2";
+
+  /* Returns a package of editable sources whose changes will be available without needing to restart the
+     nix-shell.
+     In editablePackageSources you can pass a mapping from package name to source directory to have
+     those packages available in the resulting environment, whose source changes are immediately available.
+
+  */
+  mkPoetryEditablePackage =
+    { projectDir ? null
+    , pyproject ? projectDir + "/pyproject.toml"
+    , python ? pkgs.python3
+    , pyProject ? readTOML pyproject
+      # Example: { my-app = ./src; }
+    , editablePackageSources
+    }:
+      assert editablePackageSources != { };
+      import ./editable.nix {
+        inherit pyProject python pkgs lib poetryLib editablePackageSources;
+      };
+
+  /* Returns a package containing scripts defined in tool.poetry.scripts.
+  */
+  mkPoetryScriptsPackage =
+    { projectDir ? null
+    , pyproject ? projectDir + "/pyproject.toml"
+    , python ? pkgs.python3
+    , pyProject ? readTOML pyproject
+    , scripts ? pyProject.tool.poetry.scripts
+    }:
+      assert scripts != { };
+      import ./shell-scripts.nix {
+        inherit lib python scripts;
+      };
 
   /*
      Returns an attrset { python, poetryPackages, pyProject, poetryLock } for the given pyproject/lockfile.
@@ -84,11 +117,25 @@ lib.makeScope pkgs.newScope (self: {
     , python ? pkgs.python3
     , pwd ? projectDir
     , preferWheels ? false
+      # Example: { my-app = ./src; }
+    , editablePackageSources ? { }
     , __isBootstrap ? false  # Hack: Always add Poetry as a build input unless bootstrapping
     }@attrs:
     let
       poetryPkg = poetry.override { inherit python; };
       pyProject = readTOML pyproject;
+
+      scripts = pyProject.tool.poetry.scripts or { };
+      hasScripts = scripts != { };
+      scriptsPackage = self.mkPoetryScriptsPackage {
+        inherit python scripts;
+      };
+
+      hasEditable = editablePackageSources != { };
+      editablePackage = self.mkPoetryEditablePackage {
+        inherit pyProject python editablePackageSources;
+      };
+
       poetryLock = readTOML poetrylock;
       lockFiles =
         let
@@ -180,10 +227,13 @@ lib.makeScope pkgs.newScope (self: {
 
       inputAttrs = mkInputAttrs { inherit py pyProject; attrs = { }; includeBuildSystem = false; };
 
+      storePackages = builtins.foldl' (acc: v: acc ++ v) [ ] (lib.attrValues inputAttrs);
     in
     {
       python = py;
-      poetryPackages = builtins.foldl' (acc: v: acc ++ v) [ ] (lib.attrValues inputAttrs);
+      poetryPackages = storePackages
+        ++ lib.optional hasScripts scriptsPackage
+        ++ lib.optional hasEditable editablePackage;
       poetryLock = poetryLock;
       inherit pyProject;
     };
@@ -203,38 +253,17 @@ lib.makeScope pkgs.newScope (self: {
     , pwd ? projectDir
     , python ? pkgs.python3
     , preferWheels ? false
-      # Example: { my-app = ./src; }
     , editablePackageSources ? { }
     }:
     let
-      py = self.mkPoetryPackages (
-        {
-          inherit pyproject poetrylock overrides python pwd preferWheels;
-        }
-      );
-
-      inherit (py) pyProject;
-
-      # Add executables from tool.poetry.scripts
-      scripts = pyProject.tool.poetry.scripts or { };
-      hasScripts = scripts != { };
-      scriptsPackage = import ./shell-scripts.nix {
-        inherit scripts lib;
-        inherit (py) python;
+      poetryPython = self.mkPoetryPackages {
+        inherit pyproject poetrylock overrides python pwd preferWheels editablePackageSources;
       };
 
-      hasEditable = editablePackageSources != { };
-      editablePackage = import ./editable.nix {
-        inherit pkgs lib poetryLib editablePackageSources;
-        inherit (py) pyProject python;
-      };
+      inherit (poetryPython) poetryPackages;
 
     in
-    py.python.withPackages (
-      _: py.poetryPackages
-        ++ lib.optional hasEditable editablePackage
-        ++ lib.optional hasScripts scriptsPackage
-    );
+    poetryPython.python.withPackages (_: poetryPackages);
 
   /* Creates a Python application from pyproject.toml and poetry.lock
 

pkgs/development/tools/poetry2nix/poetry2nix/lib.nix

@@ -1,4 +1,4 @@
-{ lib, pkgs }:
+{ lib, pkgs, stdenv }:
 let
   inherit (import ./semver.nix { inherit lib ireplace; }) satisfiesSemver;
   inherit (builtins) genList length;
@@ -194,6 +194,23 @@ let
         inherit src;
       };
     };
+
+  # Maps Nixpkgs CPU values to target machines known to be supported for manylinux* wheels.
+  # (a.k.a. `uname -m` output from CentOS 7)
+  #
+  # This is current as of manylinux2014 (PEP-0599), and is a superset of manylinux2010 / manylinux1.
+  # s390x is not supported in Nixpkgs, so we don't map it.
+  manyLinuxTargetMachines = {
+    x86_64 = "x86_64";
+    i686 = "i686";
+    aarch64 = "aarch64";
+    armv7l = "armv7l";
+    powerpc64 = "ppc64";
+    powerpc64le = "ppc64le";
+  };
+
+  # Machine tag for our target platform (if available)
+  targetMachine = manyLinuxTargetMachines.${stdenv.targetPlatform.parsed.cpu.name} or null;
 in
 {
   inherit
@@ -207,5 +224,6 @@ in
     cleanPythonSources
     moduleName
     getPythonVersion
+    targetMachine
     ;
 }

pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix

@@ -31,7 +31,7 @@ pythonPackages.callPackage
       inherit (poetryLib) isCompatible getManyLinuxDeps fetchFromPypi moduleName;
 
       inherit (import ./pep425.nix {
-        inherit lib python;
+        inherit lib poetryLib python;
         inherit (pkgs) stdenv;
       }) selectWheel
         ;
@@ -161,7 +161,7 @@ pythonPackages.callPackage
             builtins.fetchGit {
               inherit (source) url;
               rev = source.resolved_reference or source.reference;
-              ref = sourceSpec.branch or sourceSpec.rev or sourceSpec.tag or "HEAD";
+              ref = sourceSpec.branch or sourceSpec.rev or (if sourceSpec?tag then "refs/tags/${sourceSpec.tag}" else "HEAD");
             }
           )
         else if isUrl then

pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix

@@ -132,6 +132,18 @@ self: super:
     }
   );
 
+  datadog-lambda = super.datadog-lambda.overridePythonAttrs (old: {
+    postPatch = ''
+      substituteInPlace setup.py --replace "setuptools==" "setuptools>="
+    '';
+    buildInputs = old.buildInputs ++ [ self.setuptools ];
+  });
+
+  ddtrace = super.ddtrace.overridePythonAttrs (old: {
+    buildInputs = old.buildInputs ++
+      (pkgs.lib.optionals pkgs.stdenv.isDarwin [ pkgs.darwin.IOKit ]) ++ [ self.cython ];
+  });
+
   dictdiffer = super.dictdiffer.overridePythonAttrs (
     old: {
       buildInputs = old.buildInputs ++ [ self.pytest-runner ];
@@ -235,7 +247,7 @@ self: super:
     old:
     if old.format != "wheel" then rec {
       nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.pkg-config ];
-      buildInputs = old.buildInputs ++ [ pkgs.hdf5 self.pkg-config self.cython ];
+      buildInputs = old.buildInputs ++ [ pkgs.hdf5 self.pkgconfig self.cython ];
       configure_flags = "--hdf5=${pkgs.hdf5}";
       postConfigure = ''
         ${self.python.executable} setup.py configure ${configure_flags}
@@ -407,7 +419,7 @@ self: super:
         export LLVM_CONFIG=${pkgs.llvm}/bin/llvm-config
       '';
 
-      __impureHostDeps = pkgs.lib.optionals pkgs.stdenv.isDarwin [ "/usr/lib/libm.dylib" ];
+      __impureHostDeps = lib.optionals pkgs.stdenv.isDarwin [ "/usr/lib/libm.dylib" ];
 
       passthru = old.passthru // { llvm = pkgs.llvm; };
     }
@@ -549,6 +561,12 @@ self: super:
     }
   );
 
+  mysqlclient = super.mysqlclient.overridePythonAttrs (
+    old: {
+      buildInputs = old.buildInputs ++ [ pkgs.libmysqlclient ];
+    }
+  );
+
   netcdf4 = super.netcdf4.overridePythonAttrs (
     old: {
       buildInputs = old.buildInputs ++ [
@@ -615,6 +633,13 @@ self: super:
     }
   );
 
+  osqp = super.osqp.overridePythonAttrs (
+    old: {
+      nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.cmake ];
+      dontUseCmakeConfigure = true;
+    }
+  );
+
   parsel = super.parsel.overridePythonAttrs (
     old: rec {
       nativeBuildInputs = old.nativeBuildInputs ++ [ self.pytest-runner ];
@@ -642,6 +667,28 @@ self: super:
     }
   );
 
+  # Work around https://github.com/nix-community/poetry2nix/issues/244
+  # where git deps are not picked up as they should
+  pip =
+    if lib.versionAtLeast super.pip.version "20.3" then
+      super.pip.overridePythonAttrs
+        (old:
+          let
+            pname = "pip";
+            version = "20.2.4";
+          in
+          {
+            name = pname + "-" + version;
+            inherit version;
+            src = pkgs.fetchFromGitHub {
+              owner = "pypa";
+              repo = pname;
+              rev = version;
+              sha256 = "eMVV4ftgV71HLQsSeaOchYlfaJVgzNrwUynn3SA1/Do=";
+              name = "${pname}-${version}-source";
+            };
+          }) else super.pip;
+
   poetry-core = super.poetry-core.overridePythonAttrs (old: {
     # "Vendor" dependencies (for build-system support)
     postPatch = ''
@@ -972,6 +1019,10 @@ self: super:
 
   pytest = super.pytest.overridePythonAttrs (
     old: {
+      # Fixes https://github.com/pytest-dev/pytest/issues/7891
+      postPatch = old.postPatch or "" + ''
+        sed -i '/\[metadata\]/aversion = ${old.version}' setup.cfg
+      '';
       doCheck = false;
     }
   );
@@ -996,6 +1047,28 @@ self: super:
     }
   );
 
+  # pytest-splinter seems to put a .marker file in an empty directory
+  # presumably so it's tracked by and can be installed with MANIFEST.in, see
+  # https://github.com/pytest-dev/pytest-splinter/commit/a48eeef662f66ff9d3772af618748e73211a186b
+  #
+  # This directory then gets used as an empty initial profile directory and is
+  # zipped up. But if the .marker file is in the Nix store, it has the
+  # creation date of 1970, and Zip doesn't work with such old files, so it
+  # fails at runtime!
+  #
+  # We fix this here by just removing the file after the installation
+  #
+  # The error you get without this is:
+  #
+  # E           ValueError: ZIP does not support timestamps before 1980
+  # /nix/store/55b9ip7xkpimaccw9pa0vacy5q94f5xa-python3-3.7.6/lib/python3.7/zipfile.py:357: ValueError
+  pytest-splinter = super.pytest-splinter.overrideAttrs (old: {
+    postInstall = old.postInstall or "" + ''
+      rm $out/${super.python.sitePackages}/pytest_splinter/profiles/firefox/.marker
+    '';
+  });
+
+
   ffmpeg-python = super.ffmpeg-python.overridePythonAttrs (
     old: {
       buildInputs = old.buildInputs ++ [ self.pytest-runner ];
@@ -1168,14 +1241,16 @@ self: super:
           # is explicitly disabled with USE_CUDA=0.
           find $out -name "*.so" -exec ${pkgs.patchelf}/bin/patchelf --remove-needed libcuda.so.1 {} \;
         '';
-        buildInputs = old.buildInputs ++ lib.optionals enableCuda [
+        buildInputs = (old.buildInputs or [ ])
+          ++ [ self.typing-extensions ]
+          ++ lib.optionals enableCuda [
           pkgs.linuxPackages.nvidia_x11
           pkgs.nccl.dev
           pkgs.nccl.out
         ];
         propagatedBuildInputs = [
-          super.numpy
-          super.future
+          self.numpy
+          self.future
         ];
       })
     )
@@ -1257,15 +1332,15 @@ self: super:
         format = "wheel";
       };
       # If "wheel" is built from source
-      sourcePackage = (
+      sourcePackage = ((
         pkgs.python3.pkgs.override {
           python = self.python;
         }
-      ).wheel.overridePythonAttrs (
-        old: {
-          inherit (super.wheel) pname name version src;
-        }
-      );
+      ).wheel.override {
+        inherit (self) buildPythonPackage bootstrapped-pip setuptools;
+      }).overrideAttrs (old: {
+        inherit (super.wheel) pname name version src;
+      });
     in
     if isWheel then wheelPackage else sourcePackage;
 
@@ -1303,6 +1378,15 @@ self: super:
     }
   );
 
+  packaging = super.packaging.overridePythonAttrs (
+    old: {
+      buildInputs = old.buildInputs ++
+        # From 20.5 until 20.7, packaging used flit for packaging (heh)
+        # See https://github.com/pypa/packaging/pull/352 and https://github.com/pypa/packaging/pull/367
+        lib.optional (lib.versionAtLeast old.version "20.5" && lib.versionOlder old.version "20.8") [ self.flit-core ];
+    }
+  );
+
   supervisor = super.supervisor.overridePythonAttrs (
     old: {
       propagatedBuildInputs = old.propagatedBuildInputs ++ [
@@ -1317,4 +1401,47 @@ self: super:
       propagatedBuildInputs = old.propagatedBuildInputs ++ [ self.toolz ];
     }
   );
+
+  # For some reason the toml dependency of tqdm declared here:
+  # https://github.com/tqdm/tqdm/blob/67130a23646ae672836b971e1086b6ae4c77d930/pyproject.toml#L2
+  # is not translated correctly to a nix dependency.
+  tqdm = super.tqdm.overrideAttrs (
+    old: {
+      buildInputs = [ super.toml ] ++ old.buildInputs;
+    }
+  );
+
+  watchdog = super.watchdog.overrideAttrs (
+    old: {
+      buildInputs = old.buildInputs or [ ]
+        ++ pkgs.lib.optional pkgs.stdenv.isDarwin pkgs.darwin.apple_sdk.frameworks.CoreServices;
+    }
+  );
+
+  # pyee cannot find `vcversioner` and other "setup requirements", so it tries to
+  # download them from the internet, which only works when nix sandboxing is disabled.
+  # Additionally, since pyee uses vcversioner to specify its version, we need to do this
+  # manually specify its version.
+  pyee = super.pyee.overrideAttrs (
+    old: {
+      postPatch = old.postPatch or "" + ''
+        sed -i setup.py \
+          -e '/setup_requires/,/],/d' \
+          -e 's/vcversioner={},/version="${old.version}",/'
+      '';
+    }
+  );
+
+  # nixpkgs has setuptools_scm 4.1.2
+  # but newrelic has a seemingly unnecessary version constraint for <4
+  # So we patch that out
+  newrelic = super.newrelic.overridePythonAttrs (
+    old: {
+      postPatch = old.postPatch or "" + ''
+        substituteInPlace setup.py --replace '"setuptools_scm>=3.2,<4"' '"setuptools_scm"'
+      '';
+    }
+  );
+
+
 }

pkgs/development/tools/poetry2nix/poetry2nix/pep425.nix

@@ -1,6 +1,7 @@
-{ lib, stdenv, python, isLinux ? stdenv.isLinux }:
+{ lib, stdenv, poetryLib, python, isLinux ? stdenv.isLinux }:
 let
   inherit (lib.strings) hasSuffix hasInfix splitString removeSuffix;
+  inherit (poetryLib) targetMachine;
 
   # The 'cpxy" as determined by `python.version`
   #
@@ -72,12 +73,16 @@ let
       withPlatform =
         if isLinux
         then
-          (
-            x: x.platform == "manylinux1_${stdenv.hostPlatform.linuxArch}"
-              || x.platform == "manylinux2010_${stdenv.hostPlatform.linuxArch}"
-              || x.platform == "manylinux2014_${stdenv.hostPlatform.linuxArch}"
-              || x.platform == "any"
-          )
+          if targetMachine != null
+          then
+            (
+              x: x.platform == "manylinux1_${targetMachine}"
+                || x.platform == "manylinux2010_${targetMachine}"
+                || x.platform == "manylinux2014_${targetMachine}"
+                || x.platform == "any"
+            )
+          else
+            (x: x.platform == "any")
         else (x: hasInfix "macosx" x.platform || x.platform == "any");
       filterWheel = x:
         let

pkgs/development/tools/poetry2nix/poetry2nix/pep508.nix

@@ -1,6 +1,6 @@
 { lib, stdenv, poetryLib }: python:
 let
-  inherit (poetryLib) ireplace;
+  inherit (poetryLib) ireplace targetMachine;
 
   # Like builtins.substring but with stop being offset instead of length
   substr = start: stop: s: builtins.substring start (stop - start) s;
@@ -95,7 +95,7 @@ let
           else if stdenv.isDarwin then "darwin"
           else throw "Unsupported platform"
         );
-        platform_machine = stdenv.hostPlatform.linuxArch;
+        platform_machine = targetMachine;
         platform_python_implementation =
           let
             impl = python.passthru.implementation;
@@ -132,7 +132,7 @@ let
               mVal = ''[a-zA-Z0-9\'"_\. ]+'';
               mOp = "in|[!=<>]+";
               e = stripStr exprs.value;
-              m = builtins.map stripStr (builtins.match "^(${mVal}) *(${mOp}) *(${mVal})$" e);
+              m = builtins.map stripStr (builtins.match ''^(${mVal}) *(${mOp}) *(${mVal})$'' e);
               m0 = processVar (builtins.elemAt m 0);
               m2 = processVar (builtins.elemAt m 2);
             in

pkgs/development/tools/poetry2nix/poetry2nix/pkgs/poetry/update

@@ -3,6 +3,7 @@
 
 rev=$(curl -s https://api.github.com/repos/python-poetry/poetry/releases/latest | jq -r '.name')
 nix-prefetch-github --rev "$rev" python-poetry poetry > src.json
+echo >> src.json
 
 src=$(nix-build --no-out-link --expr 'with import <nixpkgs> {}; fetchFromGitHub (lib.importJSON ./src.json)')
 cp $src/pyproject.toml $src/poetry.lock .

pkgs/development/tools/poetry2nix/update

@@ -16,7 +16,7 @@ mv poetry2nix-master/* .
 mkdir build
 cp *.* build/
 cp -r pkgs hooks bin build/
-rm build/shell.nix build/generate.py build/overlay.nix build/flake.nix
+rm build/shell.nix build/generate.py build/overlay.nix build/flake.*
 
 cat > build/README.md << EOF
 Dont change these files here, they are maintained at https://github.com/nix-community/poetry2nix