Merge pull request #305582 from TomaSajt/dub-to-nix-git-deps

dub-to-nix,buildDubPackage: allow git-type dependencies

pkgs/build-support/dlang/builddubpackage/default.nix

@@ -2,6 +2,7 @@
   lib,
   stdenv,
   fetchurl,
+  fetchgit,
   linkFarm,
   dub,
   ldc,
@@ -43,11 +44,27 @@ let
       };
     };
 
-  lockJson = if lib.isPath dubLock then lib.importJSON dubLock else dubLock;
+  makeGitDep =
+    {
+      pname,
+      version,
+      repository,
+      sha256,
+    }:
+    {
+      inherit pname version;
+      src = fetchgit {
+        url = repository;
+        rev = version;
+        inherit sha256;
+      };
+    };
 
-  lockedDeps = lib.mapAttrsToList (
-    pname: { version, sha256 }: makeDubDep { inherit pname version sha256; }
-  ) lockJson.dependencies;
+  lockJson = if lib.isPath dubLock then lib.importJSON dubLock else dubLock;
+  depsRaw = lib.mapAttrsToList (pname: args: { inherit pname; } // args) lockJson.dependencies;
+
+  dubDeps = map makeDubDep (lib.filter (args: !(args ? repository)) depsRaw);
+  gitDeps = map makeGitDep (lib.filter (args: args ? repository) depsRaw);
 
   # a directory with multiple single element registries
   # one big directory with all .zip files leads to version parsing errors
@@ -56,7 +73,7 @@ let
     map (dep: {
       name = "${dep.pname}/${dep.pname}-${dep.version}.zip";
       path = dep.src;
-    }) lockedDeps
+    }) dubDeps
   );
 
   combinedFlags = "--skip-registry=all --compiler=${lib.getExe compiler} ${toString dubFlags}";
@@ -79,12 +96,18 @@ stdenv.mkDerivation (
         runHook preConfigure
 
         export DUB_HOME="$NIX_BUILD_TOP/.dub"
-        mkdir -p $DUB_HOME
+        mkdir -p "$DUB_HOME"
 
-        # register dependencies
+        # register dub dependencies
         ${lib.concatMapStringsSep "\n" (dep: ''
           dub fetch ${dep.pname}@${dep.version} --cache=user --skip-registry=standard --registry=file://${dubRegistryBase}/${dep.pname}
-        '') lockedDeps}
+        '') dubDeps}
+
+        # register git dependencies
+        ${lib.concatMapStringsSep "\n" (dep: ''
+          mkdir -p "$DUB_HOME/packages/${dep.pname}/${dep.version}"
+          cp -r --no-preserve=all ${dep.src} "$DUB_HOME/packages/${dep.pname}/${dep.version}/${dep.pname}"
+        '') gitDeps}
 
         runHook postConfigure
       '';

pkgs/build-support/dlang/dub-to-nix/default.nix

@@ -4,8 +4,15 @@
   makeWrapper,
   python3,
   nix,
+  nix-prefetch-git,
 }:
 
+let
+  binPath = lib.makeBinPath [
+    nix
+    nix-prefetch-git
+  ];
+in
 runCommand "dub-to-nix"
   {
     nativeBuildInputs = [ makeWrapper ];
@@ -15,5 +22,5 @@ runCommand "dub-to-nix"
     install -Dm755 ${./dub-to-nix.py} "$out/bin/dub-to-nix"
     patchShebangs "$out/bin/dub-to-nix"
     wrapProgram "$out/bin/dub-to-nix" \
-        --prefix PATH : ${lib.makeBinPath [ nix ]}
+        --prefix PATH : ${binPath}
   ''

pkgs/build-support/dlang/dub-to-nix/dub-to-nix.py

@@ -4,10 +4,13 @@ import sys
 import json
 import os
 import subprocess
+import string
+
 
 def eprint(text: str):
     print(text, file=sys.stderr)
 
+
 if not os.path.exists("dub.selections.json"):
     eprint("The file `dub.selections.json` does not exist in the current working directory")
     eprint("run `dub upgrade --annotate` to generate it")
@@ -16,24 +19,53 @@ if not os.path.exists("dub.selections.json"):
 with open("dub.selections.json") as f:
     selectionsJson = json.load(f)
 
-versionDict: dict[str, str] = selectionsJson["versions"]
+depsDict: dict = selectionsJson["versions"]
 
-for pname in versionDict:
-    version = versionDict[pname]
+# For each dependency expand non-expanded version into a dict with a "version" key
+depsDict = {pname: (versionOrDepDict if isinstance(versionOrDepDict, dict) else {"version": versionOrDepDict}) for (pname, versionOrDepDict) in depsDict.items()}
+
+# Don't process path-type selections
+depsDict = {pname: depDict for (pname, depDict) in depsDict.items() if "path" not in depDict}
+
+# Pre-validate selections before trying to fetch
+for pname in depsDict:
+    depDict = depsDict[pname]
+    version = depDict["version"]
     if version.startswith("~"):
-        eprint(f'Package "{pname}" has a branch-type version "{version}", which doesn\'t point to a fixed version')
-        eprint("You can resolve it by manually changing the required version to a fixed one inside `dub.selections.json`")
-        eprint("When packaging, you might need to create a patch for `dub.sdl` or `dub.json` to accept the changed version")
+        eprint(f'Expected version of "{pname}" to be non-branch type')
+        eprint(f'Found: "{version}"')
+        eprint("Please specify a non-branch version inside `dub.selections.json`")
+        eprint("When packaging, you might also need to patch the version value in the appropriate places (`dub.selections.json`, dub.sdl`, `dub.json`)")
         sys.exit(1)
+    if "repository" in depDict:
+        repository = depDict["repository"]
+        if not repository.startswith("git+"):
+            eprint(f'Expected repository field of "{pname}" to begin with "git+"')
+            eprint(f'Found: "{repository}"')
+            sys.exit(1)
+        if (len(version) < 7 or len(version) > 40 or not all(c in string.hexdigits for c in version)):
+            eprint(f'Expected version field of "{pname}" to begin be a valid git revision')
+            eprint(f'Found: "{version}"')
+            sys.exit(1)
 
-lockedDependenciesDict: dict[str, dict[str, str]] = {}
+lockedDepsDict: dict[str, dict[str, str]] = {}
 
-for pname in versionDict:
-    version = versionDict[pname]
-    eprint(f"Fetching {pname}@{version}")
-    url = f"https://code.dlang.org/packages/{pname}/{version}.zip"
-    command = ["nix-prefetch-url", "--type", "sha256", url]
-    sha256 = subprocess.run(command, check=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).stdout.rstrip()
-    lockedDependenciesDict[pname] = {"version": version, "sha256": sha256}
+for pname in depsDict:
+    depDict = depsDict[pname]
+    version = depDict["version"]
+    if "repository" in depDict:
+        repository = depDict["repository"]
+        strippedRepo = repository[4:]
+        eprint(f"Fetching {pname}@{version} ({strippedRepo})")
+        command = ["nix-prefetch-git", strippedRepo, version]
+        rawRes = subprocess.run(command, check=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).stdout
+        sha256 = json.loads(rawRes)["sha256"]
+        lockedDepsDict[pname] = {"version": version, "repository": repository, "sha256": sha256}
+    else:
+        eprint(f"Fetching {pname}@{version}")
+        url = f"https://code.dlang.org/packages/{pname}/{version}.zip"
+        command = ["nix-prefetch-url", "--type", "sha256", url]
+        sha256 = subprocess.run(command, check=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).stdout.rstrip()
+        lockedDepsDict[pname] = {"version": version, "sha256": sha256}
 
-print(json.dumps({"dependencies": lockedDependenciesDict}, indent=2))
+print(json.dumps({"dependencies": lockedDepsDict}, indent=2))