nodePackages: patch node2nix for npm v7+ and switch to building package set with current nodejs

nixos/doc/manual/from_md/release-notes/rl-2211.section.xml

@@ -772,6 +772,18 @@
           which no longer has a downgrade path to releases 1.2 or older.
         </para>
       </listitem>
+      <listitem>
+        <para>
+          The <literal>nodePackages</literal> package set now defaults
+          to the LTS release in the <literal>nodejs</literal> package
+          again, instead of being pinned to
+          <literal>nodejs-14_x</literal>. Several updates to node2nix
+          have been made for compatibility with newer Node.js and npm
+          versions and a new <literal>postRebuild</literal> hook has
+          been added for packages to perform extra build steps before
+          the npm install step prunes dev dependencies.
+        </para>
+      </listitem>
     </itemizedlist>
   </section>
 </section>

nixos/doc/manual/release-notes/rl-2211.section.md

@@ -254,4 +254,6 @@ Available as [services.patroni](options.html#opt-services.patroni.enable).
 
 - The `nomad` package now defaults to 1.3, which no longer has a downgrade path to releases 1.2 or older.
 
+- The `nodePackages` package set now defaults to the LTS release in the `nodejs` package again, instead of being pinned to `nodejs-14_x`. Several updates to node2nix have been made for compatibility with newer Node.js and npm versions and a new `postRebuild` hook has been added for packages to perform extra build steps before the npm install step prunes dev dependencies.
+
 <!-- To avoid merge conflicts, consider adding your item at an arbitrary place in the list instead. -->

pkgs/applications/editors/vscode/extensions/rust-analyzer/default.nix

@@ -48,7 +48,7 @@ let
     ];
 
     # Follows https://github.com/rust-lang/rust-analyzer/blob/41949748a6123fd6061eb984a47f4fe780525e63/xtask/src/dist.rs#L39-L65
-    postInstall = ''
+    postRebuild = ''
       jq '
         .version = $ENV.version |
         .releaseTag = $ENV.releaseTag |

pkgs/development/node-packages/composition.nix

@@ -2,7 +2,7 @@
 
 {pkgs ? import <nixpkgs> {
     inherit system;
-  }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-14_x"}:
+  }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-18_x"}:
 
 let
   nodeEnv = import ./node-env.nix {

pkgs/development/node-packages/generate.sh

@@ -13,7 +13,7 @@ rm -f ./node-env.nix
     -i node-packages.json \
     -o node-packages.nix \
     -c composition.nix \
-    --pkg-name nodejs-14_x
+    --pkg-name nodejs-18_x
 
 # using --no-out-link in nix-build argument would cause the
 # gc to run before the script finishes

pkgs/development/node-packages/node-env.nix

@@ -165,7 +165,11 @@ let
           if(process.argv[2] == "development") {
               replaceDependencies(packageObj.devDependencies);
           }
+          else {
+              packageObj.devDependencies = {};
+          }
           replaceDependencies(packageObj.optionalDependencies);
+          replaceDependencies(packageObj.peerDependencies);
 
           /* Write the fixed package.json file */
           fs.writeFileSync("package.json", JSON.stringify(packageObj, null, 2));
@@ -270,7 +274,7 @@ let
 
   # Reconstructs a package-lock file from the node_modules/ folder structure and package.json files with dummy sha1 hashes
   reconstructPackageLock = writeTextFile {
-    name = "addintegrityfields.js";
+    name = "reconstructpackagelock.js";
     text = ''
       var fs = require('fs');
       var path = require('path');
@@ -280,25 +284,43 @@ let
       var lockObj = {
           name: packageObj.name,
           version: packageObj.version,
-          lockfileVersion: 1,
+          lockfileVersion: 2,
           requires: true,
+          packages: {
+              "": {
+                  name: packageObj.name,
+                  version: packageObj.version,
+                  license: packageObj.license,
+                  bin: packageObj.bin,
+                  dependencies: packageObj.dependencies,
+                  engines: packageObj.engines,
+                  optionalDependencies: packageObj.optionalDependencies
+              }
+          },
           dependencies: {}
       };
 
-      function augmentPackageJSON(filePath, dependencies) {
+      function augmentPackageJSON(filePath, packages, dependencies) {
           var packageJSON = path.join(filePath, "package.json");
           if(fs.existsSync(packageJSON)) {
               var packageObj = JSON.parse(fs.readFileSync(packageJSON));
+              packages[filePath] = {
+                  version: packageObj.version,
+                  integrity: "sha1-000000000000000000000000000=",
+                  dependencies: packageObj.dependencies,
+                  engines: packageObj.engines,
+                  optionalDependencies: packageObj.optionalDependencies
+              };
               dependencies[packageObj.name] = {
                   version: packageObj.version,
                   integrity: "sha1-000000000000000000000000000=",
                   dependencies: {}
               };
-              processDependencies(path.join(filePath, "node_modules"), dependencies[packageObj.name].dependencies);
+              processDependencies(path.join(filePath, "node_modules"), packages, dependencies[packageObj.name].dependencies);
           }
       }
 
-      function processDependencies(dir, dependencies) {
+      function processDependencies(dir, packages, dependencies) {
           if(fs.existsSync(dir)) {
               var files = fs.readdirSync(dir);
 
@@ -314,23 +336,84 @@ let
                           pkgFiles.forEach(function(entry) {
                               if(stats.isDirectory()) {
                                   var pkgFilePath = path.join(filePath, entry);
-                                  augmentPackageJSON(pkgFilePath, dependencies);
+                                  augmentPackageJSON(pkgFilePath, packages, dependencies);
                               }
                           });
                       } else {
-                          augmentPackageJSON(filePath, dependencies);
+                          augmentPackageJSON(filePath, packages, dependencies);
                       }
                   }
               });
           }
       }
 
-      processDependencies("node_modules", lockObj.dependencies);
+      processDependencies("node_modules", lockObj.packages, lockObj.dependencies);
 
       fs.writeFileSync("package-lock.json", JSON.stringify(lockObj, null, 2));
     '';
   };
 
+  # Script that links bins defined in package.json to the node_modules bin directory
+  # NPM does not do this for top-level packages itself anymore as of v7
+  linkBinsScript = writeTextFile {
+    name = "linkbins.js";
+    text = ''
+      var fs = require('fs');
+      var path = require('path');
+
+      var packageObj = JSON.parse(fs.readFileSync("package.json"));
+
+      var nodeModules = Array(packageObj.name.split("/").length).fill("..").join(path.sep);
+
+      if(packageObj.bin !== undefined) {
+          fs.mkdirSync(path.join(nodeModules, ".bin"))
+
+          if(typeof packageObj.bin == "object") {
+              Object.keys(packageObj.bin).forEach(function(exe) {
+                  if(fs.existsSync(packageObj.bin[exe])) {
+                      console.log("linking bin '" + exe + "'");
+                      fs.symlinkSync(
+                          path.join("..", packageObj.name, packageObj.bin[exe]),
+                          path.join(nodeModules, ".bin", exe)
+                      );
+                  }
+                  else {
+                      console.log("skipping non-existent bin '" + exe + "'");
+                  }
+              })
+          }
+          else {
+              if(fs.existsSync(packageObj.bin)) {
+                  console.log("linking bin '" + packageObj.bin + "'");
+                  fs.symlinkSync(
+                      path.join("..", packageObj.name, packageObj.bin),
+                      path.join(nodeModules, ".bin", packageObj.name.split("/").pop())
+                  );
+              }
+              else {
+                  console.log("skipping non-existent bin '" + packageObj.bin + "'");
+              }
+          }
+      }
+      else if(packageObj.directories !== undefined && packageObj.directories.bin !== undefined) {
+          fs.mkdirSync(path.join(nodeModules, ".bin"))
+
+          fs.readdirSync(packageObj.directories.bin).forEach(function(exe) {
+              if(fs.existsSync(path.join(packageObj.directories.bin, exe))) {
+                  console.log("linking bin '" + exe + "'");
+                  fs.symlinkSync(
+                      path.join("..", packageObj.name, packageObj.directories.bin, exe),
+                      path.join(nodeModules, ".bin", exe)
+                  );
+              }
+              else {
+                  console.log("skipping non-existent bin '" + exe + "'");
+              }
+          })
+      }
+    '';
+  };
+
   prepareAndInvokeNPM = {packageName, bypassCache, reconstructLock, npmFlags, production}:
     let
       forceOfflineFlag = if bypassCache then "--offline" else "--registry http://www.example.com";
@@ -377,13 +460,18 @@ let
 
         npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${lib.optionalString production "--production"} rebuild
 
+        runHook postRebuild
+
         if [ "''${dontNpmInstall-}" != "1" ]
         then
             # NPM tries to download packages even when they already exist if npm-shrinkwrap is used.
             rm -f npm-shrinkwrap.json
 
-            npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${lib.optionalString production "--production"} install
+            npm ${forceOfflineFlag} --nodedir=${nodeSources} --no-bin-links --ignore-scripts ${npmFlags} ${lib.optionalString production "--production"} install
         fi
+
+        # Link executables defined in package.json
+        node ${linkBinsScript}
     '';
 
   # Builds and composes an NPM package including all its dependencies

pkgs/development/node-packages/overrides.nix

@@ -9,6 +9,7 @@ let
     callPackage
     fetchFromGitHub
     fetchurl
+    fetchpatch
     nixosTests;
 
   since = version: lib.versionAtLeast nodejs.version version;
@@ -92,6 +93,8 @@ final: prev: {
 
   bitwarden-cli = prev."@bitwarden/cli".override {
     name = "bitwarden-cli";
+    nativeBuildInputs = [ pkgs.pkg-config ];
+    buildInputs = with pkgs; [ pixman cairo pango ];
   };
 
   bower2nix = prev.bower2nix.override {
@@ -218,6 +221,10 @@ final: prev: {
       libsecret
       final.node-gyp-build
       final.node-pre-gyp
+
+      pixman
+      cairo
+      pango
     ] ++ lib.optionals stdenv.isDarwin [
       darwin.apple_sdk.frameworks.AppKit
       darwin.apple_sdk.frameworks.Security
@@ -314,16 +321,23 @@ final: prev: {
   };
 
   node2nix = prev.node2nix.override {
-    nativeBuildInputs = [ pkgs.buildPackages.makeWrapper ];
+    # Get latest commit for misc fixes
-    # We need to use master because of a fix that replaces git:// url to https://.
     src = fetchFromGitHub {
       owner = "svanderburg";
       repo = "node2nix";
-      rev = "68f5735f9a56737e3fedceb182705985e3ab8799";
+      rev = "026360084db8a27095aafdac7125d7f1a93046c8";
-      sha256 = "sha256-NK6gDTkGx0GG7yPTwgtFC4ttQZPfcLaLp8W8OOMO6bg=";
+      sha256 = "sha256-zO/xGG10v7HGv58RLX5SFd7QOXAL2vRxCRM8IfRZ8JA=";
     };
-
+    nativeBuildInputs = [ pkgs.buildPackages.makeWrapper ];
-    postInstall = ''
+    postInstall = let
+      # Needed to fix Node.js 16+ - PR svanderburg/node2nix#302
+      npmPatch = fetchpatch {
+        name = "emit-lockfile-v2-and-fix-bin-links-with-npmv7.patch";
+        url = "https://github.com/svanderburg/node2nix/commit/375a055041b5ee49ca5fb3f74a58ca197c90c7d5.patch";
+        hash = "sha256-uVYrXptJILojeur9s2O+J/f2vyPNCaZMn1GM/NoC5n8=";
+      };
+    in ''
+      patch -d $out/lib/node_modules/node2nix -p1 < ${npmPatch}
       wrapProgram "$out/bin/node2nix" --prefix PATH : ${lib.makeBinPath [ pkgs.nix ]}
     '';
   };
@@ -486,10 +500,18 @@ final: prev: {
 
   thelounge-theme-flat-blue = prev.thelounge-theme-flat-blue.override {
     nativeBuildInputs = [ final.node-pre-gyp ];
+    # TODO: needed until upstream pins thelounge version 4.3.1+ (which fixes dependency on old sqlite3 and transitively very old node-gyp 3.x)
+    preRebuild = ''
+      rm -r node_modules/node-gyp
+    '';
   };
 
   thelounge-theme-flat-dark = prev.thelounge-theme-flat-dark.override {
     nativeBuildInputs = [ final.node-pre-gyp ];
+    # TODO: needed until upstream pins thelounge version 4.3.1+ (which fixes dependency on old sqlite3 and transitively very old node-gyp 3.x)
+    preRebuild = ''
+      rm -r node_modules/node-gyp
+    '';
   };
 
   triton = prev.triton.override {
@@ -574,12 +596,6 @@ final: prev: {
   };
 
   wrangler = prev.wrangler.override (oldAttrs: {
-    dontNpmInstall = true;
-    nativeBuildInputs = [ pkgs.buildPackages.makeWrapper ];
-    postInstall = ''
-      makeWrapper "$out/lib/node_modules/wrangler/bin/wrangler.js" "$out/bin/wrangler" \
-        --inherit-argv0
-    '';
     meta = oldAttrs.meta // { broken = before "16.13"; };
   });
 

pkgs/top-level/all-packages.nix

@@ -6067,7 +6067,7 @@ with pkgs;
     inherit (darwin.apple_sdk.frameworks) Security;
   };
 
-  inherit (nodePackages_latest) wrangler;
+  inherit (nodePackages) wrangler;
 
   wrangler_1 = callPackage ../development/tools/wrangler_1 {
     inherit (darwin.apple_sdk.frameworks) CoreFoundation CoreServices Security;
@@ -8521,10 +8521,7 @@ with pkgs;
 
   nodePackages_latest = dontRecurseIntoAttrs nodejs_latest.pkgs;
 
-  nodePackages = (dontRecurseIntoAttrs nodejs.pkgs).override {
+  nodePackages = dontRecurseIntoAttrs nodejs.pkgs;
-    # It does not work on 16.x: https://github.com/NixOS/nixpkgs/issues/132456
-    nodejs = nodejs-14_x;
-  };
 
   node2nix = nodePackages.node2nix;