Merge pull request #150239 from pimeys/emacs-tree-sitter/link-grammars

emacsPackages.tree-sitter: fix grammar/native executor

pkgs/applications/editors/emacs/elisp-packages/manual-packages.nix

@@ -233,6 +233,9 @@
 
   sv-kalender = callPackage ./sv-kalender { };
 
+  tree-sitter-langs = callPackage ./tree-sitter-langs { final = self; };
+  tsc = callPackage ./tsc { };
+
   youtube-dl = callPackage ./youtube-dl { };
 
   # From old emacsPackages (pre emacsPackagesNg)

pkgs/applications/editors/emacs/elisp-packages/tree-sitter-langs/default-grammars.json

@@ -0,0 +1,32 @@
+[
+  "tree-sitter-agda",
+  "tree-sitter-bash",
+  "tree-sitter-c",
+  "tree-sitter-c-sharp",
+  "tree-sitter-cpp",
+  "tree-sitter-css",
+  "tree-sitter-elixir",
+  "tree-sitter-elm",
+  "tree-sitter-fluent",
+  "tree-sitter-go",
+  "tree-sitter-haskell",
+  "tree-sitter-hcl",
+  "tree-sitter-html",
+  "tree-sitter-java",
+  "tree-sitter-javascript",
+  "tree-sitter-jsdoc",
+  "tree-sitter-json",
+  "tree-sitter-julia",
+  "tree-sitter-nix",
+  "tree-sitter-ocaml",
+  "tree-sitter-php",
+  "tree-sitter-prisma",
+  "tree-sitter-python",
+  "tree-sitter-ruby",
+  "tree-sitter-rust",
+  "tree-sitter-scala",
+  "tree-sitter-swift",
+  "tree-sitter-typescript",
+  "tree-sitter-verilog",
+  "tree-sitter-zig"
+]

pkgs/applications/editors/emacs/elisp-packages/tree-sitter-langs/default.nix

@@ -0,0 +1,44 @@
+{ lib
+, pkgs
+, symlinkJoin
+, fetchzip
+, melpaBuild
+, stdenv
+, fetchFromGitHub
+, writeText
+, melpaStablePackages
+, runCommand
+, tree-sitter-grammars
+, plugins ? map (g: tree-sitter-grammars.${g}) (lib.importJSON ./default-grammars.json)
+, final
+}:
+
+let
+  inherit (melpaStablePackages) tree-sitter-langs;
+
+  libSuffix = if stdenv.isDarwin then "dylib" else "so";
+  soName = g: lib.removeSuffix "-grammar" (lib.removePrefix "tree-sitter-" g.pname) + "." + libSuffix;
+
+  grammarDir = runCommand "emacs-tree-sitter-grammars" {
+    # Fake same version number as upstream language bundle to prevent triggering runtime downloads
+    inherit (tree-sitter-langs) version;
+  } (''
+    install -d $out/langs/bin
+    echo -n $version > $out/langs/bin/BUNDLE-VERSION
+  '' + lib.concatStringsSep "\n" (map (
+    g: "ln -s ${g}/parser $out/langs/bin/${soName g}") plugins
+  ));
+
+in
+melpaStablePackages.tree-sitter-langs.overrideAttrs(old: {
+  postPatch = old.postPatch or "" + ''
+    substituteInPlace ./tree-sitter-langs-build.el \
+    --replace "tree-sitter-langs-grammar-dir tree-sitter-langs--dir"  "tree-sitter-langs-grammar-dir \"${grammarDir}/langs\""
+  '';
+
+  passthru = old.passthru or {} // {
+    inherit plugins;
+    withPlugins = fn: final.tree-sitter-langs.override { plugins = fn tree-sitter-grammars; };
+  };
+
+})

pkgs/applications/editors/emacs/elisp-packages/tree-sitter-langs/update-defaults.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env nix-shell
+#! nix-shell ../../../../../../. -i python3 -p python3 -p nix
+from os.path import (
+    dirname,
+    abspath,
+    join,
+)
+from typing import (
+    List,
+    Any,
+)
+import subprocess
+import json
+import sys
+import os
+
+
+def fmt_grammar(grammar: str) -> str:
+    return "tree-sitter-" + grammar
+
+
+def eval_expr(nixpkgs: str, expr: str) -> Any:
+    p = subprocess.run(
+        [
+            "nix-instantiate",
+            "--json",
+            "--eval",
+            "--expr",
+            ("with import %s {}; %s" % (nixpkgs, expr)),
+        ],
+        check=True,
+        stdout=subprocess.PIPE,
+    )
+    return json.loads(p.stdout)
+
+
+def check_grammar_exists(nixpkgs: str, grammar: str) -> bool:
+    return eval_expr(
+        nixpkgs, f'lib.hasAttr "{fmt_grammar(grammar)}" tree-sitter-grammars'
+    )
+
+
+def build_attr(nixpkgs, attr: str) -> str:
+    return (
+        subprocess.run(
+            ["nix-build", "--no-out-link", nixpkgs, "-A", attr],
+            check=True,
+            stdout=subprocess.PIPE,
+        )
+        .stdout.decode()
+        .strip()
+    )
+
+
+if __name__ == "__main__":
+    cwd = dirname(abspath(__file__))
+    nixpkgs = abspath(join(cwd, "../../../../../.."))
+
+    src_dir = build_attr(nixpkgs, "emacs.pkgs.tree-sitter-langs.src")
+
+    existing: List[str] = []
+
+    grammars = os.listdir(join(src_dir, "repos"))
+    for g in grammars:
+        exists = check_grammar_exists(nixpkgs, g)
+        if exists:
+            existing.append(fmt_grammar(g))
+        else:
+            sys.stderr.write("Missing grammar: " + fmt_grammar(g) + "\n")
+            sys.stderr.flush()
+
+    with open(join(cwd, "default-grammars.json"), mode="w") as f:
+        json.dump(sorted(existing), f, indent=2)
+        f.write("\n")

pkgs/applications/editors/emacs/elisp-packages/tsc/default.nix

@@ -0,0 +1,89 @@
+{ lib
+, symlinkJoin
+, melpaBuild
+, fetchFromGitHub
+, rustPlatform
+, writeText
+, clang
+, llvmPackages
+
+, runtimeShell
+, writeScript
+, python3
+, nix-prefetch-github
+, nix
+}:
+
+let
+
+  srcMeta = lib.importJSON ./src.json;
+  inherit (srcMeta) version;
+
+  src = fetchFromGitHub srcMeta.src;
+
+  tsc = melpaBuild {
+    inherit src;
+    inherit version;
+
+    pname = "tsc";
+    commit = version;
+
+    sourceRoot = "source/core";
+
+    recipe = writeText "recipe" ''
+      (tsc
+      :repo "emacs-tree-sitter/elisp-tree-sitter"
+      :fetcher github)
+    '';
+  };
+
+  tsc-dyn = rustPlatform.buildRustPackage {
+    inherit version;
+    inherit src;
+
+    pname = "tsc-dyn";
+
+    nativeBuildInputs = [ clang ];
+    sourceRoot = "source/core";
+
+    configurePhase = ''
+      export LIBCLANG_PATH="${llvmPackages.libclang.lib}/lib"
+    '';
+
+    postInstall = ''
+      LIB=($out/lib/libtsc_dyn.*)
+      TSC_PATH=$out/share/emacs/site-lisp/elpa/tsc-${version}
+      install -d $TSC_PATH
+      install -m444 $out/lib/libtsc_dyn.* $TSC_PATH/''${LIB/*libtsc_/tsc-}
+      echo -n $version > $TSC_PATH/DYN-VERSION
+      rm -r $out/lib
+    '';
+
+    inherit (srcMeta) cargoSha256;
+  };
+
+in symlinkJoin {
+  name = "tsc-${version}";
+  paths = [ tsc tsc-dyn ];
+
+  passthru = {
+    updateScript = let
+      pythonEnv = python3.withPackages(ps: [ ps.requests ]);
+    in writeScript "tsc-update" ''
+      #!${runtimeShell}
+      set -euo pipefail
+      export PATH=${lib.makeBinPath [
+        nix-prefetch-github
+        nix
+        pythonEnv
+      ]}:$PATH
+      exec python3 ${builtins.toString ./update.py} ${builtins.toString ./.}
+    '';
+  };
+
+  meta = {
+    description = "The core APIs of the Emacs binding for tree-sitter.";
+    license = lib.licenses.mit;
+    maintainers = with lib.maintainers; [ pimeys ];
+  };
+}

pkgs/applications/editors/emacs/elisp-packages/tsc/src.json

@@ -0,0 +1,10 @@
+{
+  "src": {
+    "owner": "emacs-tree-sitter",
+    "repo": "elisp-tree-sitter",
+    "rev": "909717c685ff5a2327fa2ca8fb8a25216129361c",
+    "sha256": "LrakDpP3ZhRQqz47dPcyoQnu5lROdaNlxGaQfQT6u+k="
+  },
+  "version": "0.18.0",
+  "cargoSha256": "sha256-IRCZqszBkGF8anF/kpcPOzHdOP4lAtJBAp6FS5tAOx8="
+}

pkgs/applications/editors/emacs/elisp-packages/tsc/update.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+from textwrap import dedent
+from os.path import (
+    abspath,
+    dirname,
+    join,
+)
+from typing import (
+    Dict,
+    Any,
+)
+import subprocess
+import tempfile
+import json
+import sys
+import re
+
+import requests
+
+
+def eval_drv(nixpkgs: str, expr: str) -> Any:
+    expr = "\n".join(
+        (
+            "with (import %s {});" % nixpkgs,
+            expr,
+        )
+    )
+
+    with tempfile.NamedTemporaryFile(mode="w") as f:
+        f.write(dedent(expr))
+        f.flush()
+        p = subprocess.run(
+            ["nix-instantiate", "--json", f.name], stdout=subprocess.PIPE, check=True
+        )
+
+    return p.stdout.decode().strip()
+
+
+def get_src(tag_name: str) -> Dict[str, str]:
+    p = subprocess.run(
+        [
+            "nix-prefetch-github",
+            "--rev",
+            tag_name,
+            "--json",
+            "emacs-tree-sitter",
+            "elisp-tree-sitter",
+        ],
+        stdout=subprocess.PIPE,
+        check=True,
+    )
+    src = json.loads(p.stdout)
+
+    fields = ["owner", "repo", "rev", "sha256"]
+
+    return {f: src[f] for f in fields}
+
+
+def get_cargo_sha256(drv_path: str):
+    # Note: No check=True since we expect this command to fail
+    p = subprocess.run(["nix-store", "-r", drv_path], stderr=subprocess.PIPE)
+
+    stderr = p.stderr.decode()
+    lines = iter(stderr.split("\n"))
+
+    for l in lines:
+        if l.startswith("error: hash mismatch in fixed-output derivation"):
+            break
+    else:
+        raise ValueError("Did not find expected hash mismatch message")
+
+    for l in lines:
+        m = re.match(r"\s+got:\s+(.+)$", l)
+        if m:
+            return m.group(1)
+
+    raise ValueError("Could not extract actual sha256 hash: ", stderr)
+
+
+if __name__ == "__main__":
+    cwd = sys.argv[1]
+
+    nixpkgs = abspath(join(cwd, "../../../../../.."))
+
+    tag_name = requests.get(
+        "https://api.github.com/repos/emacs-tree-sitter/elisp-tree-sitter/releases/latest"
+    ).json()["tag_name"]
+
+    src = get_src(tag_name)
+
+    with tempfile.NamedTemporaryFile(mode="w") as f:
+        json.dump(src, f)
+        f.flush()
+
+        drv_path = eval_drv(
+            nixpkgs,
+            """
+        rustPlatform.buildRustPackage {
+          pname = "tsc-dyn";
+          version = "%s";
+          nativeBuildInputs = [ clang ];
+          src = fetchFromGitHub (lib.importJSON %s);
+          sourceRoot = "source/core";
+          cargoSha256 = lib.fakeSha256;
+        }
+        """
+            % (tag_name, f.name),
+        )
+
+    cargo_sha256 = get_cargo_sha256(drv_path)
+
+    with open(join(cwd, "src.json"), mode="w") as f:
+        json.dump(
+            {
+                "src": src,
+                "version": tag_name,
+                "cargoSha256": cargo_sha256,
+            },
+            f,
+            indent=2,
+        )
+        f.write("\n")