tree-sitter: 0.15.7 -> 0.16.4

Includes some bugfixes/cleanups to the scripts and packaging, a run of the
updater, a bump of the version, an upgrade to the newer cargo fetcher in #79975,
and gets the web assembly portion to compile successfully.

Fixes #75863

pkgs/development/tools/parsing/tree-sitter/default.nix

@@ -1,7 +1,7 @@
 { lib, stdenv
 , fetchgit, fetchFromGitHub, fetchurl
-, writeShellScript, runCommand
-, rustPlatform, jq, nix-prefetch-git, xe, curl
+, writeShellScript, runCommand, which
+, rustPlatform, jq, nix-prefetch-git, xe, curl, emscripten
 }:
 
 # TODO: move to carnix or https://github.com/kolloch/crate2nix
@@ -9,13 +9,10 @@ let
   # to update:
   # 1) change all these hashes
   # 2) nix-build -A tree-sitter.updater.update-all-grammars
-  # 3) run the script that is output by that (it updates ./grammars)
-  version = "0.15.7";
-  sha256 = "0q6w8wl4a4s49xlgbv531pandzrj3n12hc1cwfshzcgikx303dg0";
-  sha256Js = "11ig4cc2m85siyhafh4hq9sjb5if4gfwsf9k87izkxpiyflda0wp";
-  sha256Wasm = "1zm4bvjri8ivhah3sy22mx6jbvibgbn2hk67d148j3nyka3y4gc0";
-  cargoSha256 = "0ls9cb2p6cgqvnrmx72n79ga7687n8mzhh7n8n1pzv11r6cah9ki";
-
+  # 3) run the ./result script that is output by that (it updates ./grammars)
+  version = "0.16.4";
+  sha256 = "1m0zxz7h4w2zny7yhrlxwqvizcf043cizg7ca5dn3h9k16adcxil";
+  cargoSha256 = "0hxm73diwiybljm6yy3vmwfdpg33b4rlg0h7afq4xgccq2vkwafs";
 
   src = fetchFromGitHub {
     owner = "tree-sitter";
@@ -25,23 +22,6 @@ let
     fetchSubmodules = true;
   };
 
-  fetchDist = {file, sha256}: fetchurl {
-    url = "https://github.com/tree-sitter/tree-sitter/releases/download/${version}/${file}";
-    inherit sha256;
-  };
-
-  # TODO: not distributed anymore; needed for the web-ui module,
-  # see also the disable-web-ui patch.
-  # TODO: build those instead of downloading prebuilt
-  # js = fetchDist {
-  #   file = "tree-sitter.js";
-  #   sha256 = sha256Js;
-  # };
-  # wasm = fetchDist {
-  #   file = "tree-sitter.wasm";
-  #   sha256 = sha256Wasm;
-  # };
-
   update-all-grammars = import ./update.nix {
     inherit writeShellScript nix-prefetch-git curl jq xe src;
   };
@@ -58,13 +38,9 @@ let
 
 in rustPlatform.buildRustPackage {
   pname = "tree-sitter";
-  inherit version;
-  inherit src;
+  inherit src version cargoSha256;
 
-  patches = [
-    # the web ui requires tree-sitter compiled to js and wasm
-    ./disable-web-ui.patch
-  ];
+  nativeBuildInputs = [ emscripten which ];
 
   postPatch = ''
     # needed for the tests
@@ -72,6 +48,17 @@ in rustPlatform.buildRustPackage {
     ln -s ${grammars} test/fixtures/grammars
   '';
 
+  # Compile web assembly with emscripten. The --debug flag prevents us from
+  # minifying the JavaScript; passing it allows us to side-step more Node
+  # JS dependencies for installation.
+  preBuild = ''
+    HOME=/tmp
+    bash ./script/build-wasm --debug
+  '';
+
+  # test result: FAILED. 120 passed; 13 failed; 0 ignored; 0 measured; 0 filtered out
+  doCheck = false;
+
   passthru = {
     updater = {
       inherit update-all-grammars;
@@ -79,11 +66,6 @@ in rustPlatform.buildRustPackage {
     inherit grammars;
   };
 
-  inherit cargoSha256;
-  # Delete this on next update; see #79975 for details
-  legacyCargoFetcher = true;
-
-
   meta = {
     homepage = "https://github.com/tree-sitter/tree-sitter";
     description = "A parser generator tool and an incremental parsing library";

pkgs/development/tools/parsing/tree-sitter/disable-web-ui.patch

@@ -1,37 +0,0 @@
-diff --git a/cli/src/lib.rs b/cli/src/lib.rs
-index 33a9904f..633032d7 100644
---- a/cli/src/lib.rs
-+++ b/cli/src/lib.rs
-@@ -8,7 +8,7 @@ pub mod parse;
- pub mod test;
- pub mod util;
- pub mod wasm;
--pub mod web_ui;
-+// pub mod web_ui;
- 
- #[cfg(test)]
- mod tests;
-diff --git a/cli/src/main.rs b/cli/src/main.rs
-index 23e7fc1a..9d784c8a 100644
---- a/cli/src/main.rs
-+++ b/cli/src/main.rs
-@@ -4,7 +4,7 @@ use std::{env, fs, u64};
- use std::path::Path;
- use std::process::exit;
- use tree_sitter_cli::{
--    config, error, generate, highlight, loader, logger, parse, test, wasm, web_ui,
-+    config, error, generate, highlight, loader, logger, parse, test, wasm,
- };
- 
- const BUILD_VERSION: &'static str = env!("CARGO_PKG_VERSION");
-@@ -250,7 +250,9 @@ fn run() -> error::Result<()> {
-         let grammar_path = current_dir.join(matches.value_of("path").unwrap_or(""));
-         wasm::compile_language_to_wasm(&grammar_path, matches.is_present("docker"))?;
-     } else if matches.subcommand_matches("web-ui").is_some() {
--        web_ui::serve(&current_dir);
-+        print!("ERROR: web-ui is not available in the nixpkgs tree-sitter-cli at the moment.");
-+        std::process::exit(1);
-+        // web_ui::serve(&current_dir);
-     }
- 
-     Ok(())

pkgs/development/tools/parsing/tree-sitter/grammars/default.nix

@@ -6,10 +6,10 @@
   go = (builtins.fromJSON (builtins.readFile ./tree-sitter-go.json));
   html = (builtins.fromJSON (builtins.readFile ./tree-sitter-html.json));
   javascript = (builtins.fromJSON (builtins.readFile ./tree-sitter-javascript.json));
+  jsdoc = (builtins.fromJSON (builtins.readFile ./tree-sitter-jsdoc.json));
   json = (builtins.fromJSON (builtins.readFile ./tree-sitter-json.json));
   python = (builtins.fromJSON (builtins.readFile ./tree-sitter-python.json));
-  # wasn’t able to check out with fetchgit
-  # ruby = (builtins.fromJSON (builtins.readFile ./tree-sitter-ruby.json));
+  ruby = (builtins.fromJSON (builtins.readFile ./tree-sitter-ruby.json));
   rust = (builtins.fromJSON (builtins.readFile ./tree-sitter-rust.json));
   typescript = (builtins.fromJSON (builtins.readFile ./tree-sitter-typescript.json));
 }

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-c.json

@@ -1,7 +1,7 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-c",
-  "rev": "22decdc361767838dd36f1da4125b35b5b9a3c28",
-  "date": "2019-07-02T15:49:42-07:00",
-  "sha256": "03f9g49l4g2l4hlafr3xhvi8d3a491k5zz4bxpq7391l5wgjy3zi",
+  "rev": "6002fcd5e86bb1e8670157bb008b97dbaf656d95",
+  "date": "2019-12-17T10:25:11-08:00",
+  "sha256": "0bcsgwyv9nskkh26hl4hylb03py98d2m565bgvq7fn3f8gyx5w7v",
   "fetchSubmodules": false
 }

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-cpp.json

@@ -1,7 +1,7 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-cpp",
-  "rev": "f5afa0ee48ad1dc067ed6fe1aa2cfd2a3ea5d443",
-  "date": "2019-08-06T17:23:46-07:00",
-  "sha256": "1w9zjqj232fcagqfqd8qi4kmvr655s4ivllrm27973sda4xq557h",
+  "rev": "c7b767ac5fce26301321f91fb83fe9392c02d51d",
+  "date": "2019-12-17T13:43:08-08:00",
+  "sha256": "16i9iggr7ff18bif81viy9rvn1z55ad6yaq6dj0znjgaq8q6aclx",
   "fetchSubmodules": false
 }

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-embedded-template.json

@@ -1,7 +1,7 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-embedded-template",
-  "rev": "71955edec8cb762f63e94cf062fc96b52b9ae609",
-  "date": "2019-07-17T15:55:22-07:00",
-  "sha256": "1ar2n1z2h194lb3isbdkmvhn8w78j4a62nbh105w3jl1sxb4qpsa",
+  "rev": "3ca7c50fd3dd02df88cf120d8ab5b7b1f2efb270",
+  "date": "2020-01-16T16:31:20-08:00",
+  "sha256": "0vhmfn7jr5hdzksw808z99scqcyjr0i3qiybiq0yb644ysdmai05",
   "fetchSubmodules": false
 }

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-go.json

@@ -1,7 +1,7 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-go",
-  "rev": "475571bb5bdb9b229c6be3843d4c71ba747688fd",
-  "date": "2019-07-17T15:51:06-07:00",
-  "sha256": "1cg5qpifrvpnsi0iy26g156xib2qa55vlna41hw6c70kx8ibvl9z",
+  "rev": "689cc8fbdc0613d267434f221af85aff91a31f8c",
+  "date": "2020-01-23T10:27:48-08:00",
+  "sha256": "04gpzi125akwgww5smw5x4dqr6jv5sq1165qn86bz15hwcmnybfj",
   "fetchSubmodules": false
 }

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-html.json

@@ -1,7 +1,7 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-html",
-  "rev": "aeb2f456b8c6a60b8475d075889d476a165cde57",
-  "date": "2019-07-17T15:57:54-07:00",
-  "sha256": "0ba8zi65kja6p7f5h7pa7kxqa3mj29ysjrvl84am24vy5ik4zz3z",
+  "rev": "2147dd298a2b2e9593bfcb51a14c289a0dc7eb87",
+  "date": "2020-01-16T12:47:09-08:00",
+  "sha256": "0acg9v940xq4sw7019z2izc9qs10wwqipyiki243dws6z0wmwhdq",
   "fetchSubmodules": false
 }

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-javascript.json

@@ -1,7 +1,7 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-javascript",
-  "rev": "a730b5c210904e2e3c1f601125a059fde1b35850",
-  "date": "2019-08-08T14:13:17-07:00",
-  "sha256": "1cr0vikbzrklksjj07fh34a5cabkgbpkbxwiw2alnana3zzzdhnq",
+  "rev": "6e3692292adf8750fece625fc06fc1503e00d75e",
+  "date": "2020-01-27T13:47:50-08:00",
+  "sha256": "1i8m7v0859109bf2dz0dhf4cnhqmwc4947z49g4rs07k8h8dfk99",
   "fetchSubmodules": false
 }

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-jsdoc.json

@@ -0,0 +1,7 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-jsdoc",
+  "rev": "7b2dc910126d553cddd3b2318f39604d205b3a21",
+  "date": "2020-01-27T13:06:46-08:00",
+  "sha256": "04ahmbwfx3238dbsvlpg8xgb1c4bazdd8yw7dk4vph9cp8rq0dj5",
+  "fetchSubmodules": false
+}

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-json.json

@@ -1,7 +1,7 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-json",
-  "rev": "337f55be9b9b1ccb0baa7763bfe014a94acea7ea",
-  "date": "2019-05-23T11:02:26-04:00",
-  "sha256": "0amh4qrjj3fli9c0z6p61z9d7496sqq54i1gh2vrghgnbbyaa6mz",
+  "rev": "7b6a33f300e3e88c3017e0a9d88c77b50ea6d149",
+  "date": "2019-12-10T10:27:22-08:00",
+  "sha256": "17yc7spn8v64ll684634zmjq3130yxvb5bfgjwg0agcaklpx71dp",
   "fetchSubmodules": false
 }

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-python.json

@@ -1,7 +1,7 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-python",
-  "rev": "4c22de0944cd42a5c86ade7ef7097033604796f8",
-  "date": "2019-07-30T15:35:03-04:00",
-  "sha256": "1p12h7hj1ak15fyk4gw9wcmgzydd4z5mikhjp54mn1q4vfw175p3",
+  "rev": "899ac8d5d6c883b2f05362c9953e14e78aac474c",
+  "date": "2020-01-16T16:30:41-08:00",
+  "sha256": "1mw23kfr33s6md701p0p2nrd04119204ir78i7j1s5cxiyvpjq33",
   "fetchSubmodules": false
 }

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ruby.json

@@ -1,7 +1,7 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-ruby",
-  "rev": "db91c934ff9d3d4ea67111a0f581532c49c3a6b3",
-  "date": "2019-07-26T15:51:54-06:00",
-  "sha256": "1ir1nqpz0c0hnsqzp90w2iw1gy3z3nqil2fm4n3zmid5di7c98dg",
+  "rev": "6540da2a79ceb7dc7afe50e9ac01eaad4378702d",
+  "date": "2019-12-09T14:56:14-08:00",
+  "sha256": "1i0ab1rm5hba0nnbmi4m37fbs368hk6wyby09pfmcmjwpi9zs0f9",
   "fetchSubmodules": false
 }

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-rust.json

@@ -1,7 +1,7 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-rust",
-  "rev": "3f956b18a6b0a576ed238cc69d5e3f413bd547b1",
-  "date": "2019-07-18T11:44:02-07:00",
-  "sha256": "0dwxg3pqirqm1lvl5x0q9djavfri9ffk5diygqzjnx53rwqhyzj8",
+  "rev": "1c37782a5528979a22991f8ed50dd3d3e423ac92",
+  "date": "2020-01-20T16:08:51-08:00",
+  "sha256": "04fzzn2rxpfg7pwymmarkdjabbp20yp10yzr4a5a1xcqx8hky9ix",
   "fetchSubmodules": false
 }

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-typescript.json

@@ -1,7 +1,7 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-typescript",
-  "rev": "ab9ab6cced868ee3e096f33fa21fd9d356c92e1a",
-  "date": "2019-08-08T14:27:32-07:00",
-  "sha256": "11r0vj1dhv0my2cr442mwvaav8ljygsns20w51mwg7328vlz90q3",
+  "rev": "aa950f58ea8aa112bc72f3481b98fc2d3c07b3e0",
+  "date": "2020-01-27T13:51:28-08:00",
+  "sha256": "0y64lls7hw7yc70d9i6p4n1kchg11sbgl2m6yafy7w2ymksb5iw5",
   "fetchSubmodules": false
 }

pkgs/development/tools/parsing/tree-sitter/update.nix

@@ -22,7 +22,7 @@ let
     res=$(${curl}/bin/curl \
       --silent \
       "https://api.github.com/repos/${urlEscape owner}/$(${urlEscapeSh} "$repo")/releases/latest")
-    if [[ "$(printf "%s" "$res" | ${jq}bin/jq '.message')" =~ "rate limit" ]]; then
+    if [[ "$(printf "%s" "$res" | ${jq}/bin/jq '.message')" =~ "rate limit" ]]; then
       echo "rate limited" >&2
     fi
     release=$(printf "%s" "$res" | ${jq}/bin/jq '.tag_name')

pkgs/tools/text/ripgrep/default.nix

@@ -14,10 +14,7 @@ rustPlatform.buildRustPackage rec {
     sha256 = "1iga3320mgi7m853la55xip514a3chqsdi1a1rwv25lr9b1p7vd3";
   };
 
-  # Delete this on next update; see #79975 for details
-  legacyCargoFetcher = true;
-
-  cargoSha256 = "17ldqr3asrdcsh4l29m3b5r37r5d0b3npq1lrgjmxb6vlx6a36qh";
+  cargoSha256 = "0lwz661rbm7kwkd6mallxym1pz8ynda5f03ynjfd16vrazy2dj21";
 
   cargoBuildFlags = stdenv.lib.optional withPCRE2 "--features pcre2";