From 9131efe52d72ffb35b56e7d943b522305266ce15 Mon Sep 17 00:00:00 2001 From: Benjamin Hipple Date: Sat, 15 Feb 2020 00:11:00 -0500 Subject: [PATCH] tree-sitter: 0.15.7 -> 0.16.4 Includes some bugfixes/cleanups to the scripts and packaging, a run of the updater, a bump of the version, an upgrade to the newer cargo fetcher in #79975, and gets the web assembly portion to compile successfully. Fixes #75863 --- .../tools/parsing/tree-sitter/default.nix | 56 +++++++------------ .../parsing/tree-sitter/disable-web-ui.patch | 37 ------------ .../parsing/tree-sitter/grammars/default.nix | 4 +- .../tree-sitter/grammars/tree-sitter-c.json | 6 +- .../tree-sitter/grammars/tree-sitter-cpp.json | 6 +- .../tree-sitter-embedded-template.json | 6 +- .../tree-sitter/grammars/tree-sitter-go.json | 6 +- .../grammars/tree-sitter-html.json | 6 +- .../grammars/tree-sitter-javascript.json | 6 +- .../grammars/tree-sitter-jsdoc.json | 7 +++ .../grammars/tree-sitter-json.json | 6 +- .../grammars/tree-sitter-python.json | 6 +- .../grammars/tree-sitter-ruby.json | 6 +- .../grammars/tree-sitter-rust.json | 6 +- .../grammars/tree-sitter-typescript.json | 6 +- .../tools/parsing/tree-sitter/update.nix | 2 +- pkgs/tools/text/ripgrep/default.nix | 5 +- 17 files changed, 63 insertions(+), 114 deletions(-) delete mode 100644 pkgs/development/tools/parsing/tree-sitter/disable-web-ui.patch create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-jsdoc.json diff --git a/pkgs/development/tools/parsing/tree-sitter/default.nix b/pkgs/development/tools/parsing/tree-sitter/default.nix index 2ef86070fccd..0e69fa3259f8 100644 --- a/pkgs/development/tools/parsing/tree-sitter/default.nix +++ b/pkgs/development/tools/parsing/tree-sitter/default.nix @@ -1,7 +1,7 @@ { lib, stdenv , fetchgit, fetchFromGitHub, fetchurl -, writeShellScript, runCommand -, rustPlatform, jq, nix-prefetch-git, xe, curl +, writeShellScript, runCommand, which +, rustPlatform, jq, nix-prefetch-git, xe, curl, emscripten }: # TODO: move to carnix or https://github.com/kolloch/crate2nix @@ -9,13 +9,10 @@ let # to update: # 1) change all these hashes # 2) nix-build -A tree-sitter.updater.update-all-grammars - # 3) run the script that is output by that (it updates ./grammars) - version = "0.15.7"; - sha256 = "0q6w8wl4a4s49xlgbv531pandzrj3n12hc1cwfshzcgikx303dg0"; - sha256Js = "11ig4cc2m85siyhafh4hq9sjb5if4gfwsf9k87izkxpiyflda0wp"; - sha256Wasm = "1zm4bvjri8ivhah3sy22mx6jbvibgbn2hk67d148j3nyka3y4gc0"; - cargoSha256 = "0ls9cb2p6cgqvnrmx72n79ga7687n8mzhh7n8n1pzv11r6cah9ki"; - + # 3) run the ./result script that is output by that (it updates ./grammars) + version = "0.16.4"; + sha256 = "1m0zxz7h4w2zny7yhrlxwqvizcf043cizg7ca5dn3h9k16adcxil"; + cargoSha256 = "0hxm73diwiybljm6yy3vmwfdpg33b4rlg0h7afq4xgccq2vkwafs"; src = fetchFromGitHub { owner = "tree-sitter"; @@ -25,23 +22,6 @@ let fetchSubmodules = true; }; - fetchDist = {file, sha256}: fetchurl { - url = "https://github.com/tree-sitter/tree-sitter/releases/download/${version}/${file}"; - inherit sha256; - }; - - # TODO: not distributed anymore; needed for the web-ui module, - # see also the disable-web-ui patch. - # TODO: build those instead of downloading prebuilt - # js = fetchDist { - # file = "tree-sitter.js"; - # sha256 = sha256Js; - # }; - # wasm = fetchDist { - # file = "tree-sitter.wasm"; - # sha256 = sha256Wasm; - # }; - update-all-grammars = import ./update.nix { inherit writeShellScript nix-prefetch-git curl jq xe src; }; @@ -58,13 +38,9 @@ let in rustPlatform.buildRustPackage { pname = "tree-sitter"; - inherit version; - inherit src; + inherit src version cargoSha256; - patches = [ - # the web ui requires tree-sitter compiled to js and wasm - ./disable-web-ui.patch - ]; + nativeBuildInputs = [ emscripten which ]; postPatch = '' # needed for the tests @@ -72,6 +48,17 @@ in rustPlatform.buildRustPackage { ln -s ${grammars} test/fixtures/grammars ''; + # Compile web assembly with emscripten. The --debug flag prevents us from + # minifying the JavaScript; passing it allows us to side-step more Node + # JS dependencies for installation. + preBuild = '' + HOME=/tmp + bash ./script/build-wasm --debug + ''; + + # test result: FAILED. 120 passed; 13 failed; 0 ignored; 0 measured; 0 filtered out + doCheck = false; + passthru = { updater = { inherit update-all-grammars; @@ -79,11 +66,6 @@ in rustPlatform.buildRustPackage { inherit grammars; }; - inherit cargoSha256; - # Delete this on next update; see #79975 for details - legacyCargoFetcher = true; - - meta = { homepage = "https://github.com/tree-sitter/tree-sitter"; description = "A parser generator tool and an incremental parsing library"; diff --git a/pkgs/development/tools/parsing/tree-sitter/disable-web-ui.patch b/pkgs/development/tools/parsing/tree-sitter/disable-web-ui.patch deleted file mode 100644 index 6065aa9e7fdb..000000000000 --- a/pkgs/development/tools/parsing/tree-sitter/disable-web-ui.patch +++ /dev/null @@ -1,37 +0,0 @@ -diff --git a/cli/src/lib.rs b/cli/src/lib.rs -index 33a9904f..633032d7 100644 ---- a/cli/src/lib.rs -+++ b/cli/src/lib.rs -@@ -8,7 +8,7 @@ pub mod parse; - pub mod test; - pub mod util; - pub mod wasm; --pub mod web_ui; -+// pub mod web_ui; - - #[cfg(test)] - mod tests; -diff --git a/cli/src/main.rs b/cli/src/main.rs -index 23e7fc1a..9d784c8a 100644 ---- a/cli/src/main.rs -+++ b/cli/src/main.rs -@@ -4,7 +4,7 @@ use std::{env, fs, u64}; - use std::path::Path; - use std::process::exit; - use tree_sitter_cli::{ -- config, error, generate, highlight, loader, logger, parse, test, wasm, web_ui, -+ config, error, generate, highlight, loader, logger, parse, test, wasm, - }; - - const BUILD_VERSION: &'static str = env!("CARGO_PKG_VERSION"); -@@ -250,7 +250,9 @@ fn run() -> error::Result<()> { - let grammar_path = current_dir.join(matches.value_of("path").unwrap_or("")); - wasm::compile_language_to_wasm(&grammar_path, matches.is_present("docker"))?; - } else if matches.subcommand_matches("web-ui").is_some() { -- web_ui::serve(¤t_dir); -+ print!("ERROR: web-ui is not available in the nixpkgs tree-sitter-cli at the moment."); -+ std::process::exit(1); -+ // web_ui::serve(¤t_dir); - } - - Ok(()) diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix b/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix index b6451bce7282..ef0db9d68a9b 100644 --- a/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix @@ -6,10 +6,10 @@ go = (builtins.fromJSON (builtins.readFile ./tree-sitter-go.json)); html = (builtins.fromJSON (builtins.readFile ./tree-sitter-html.json)); javascript = (builtins.fromJSON (builtins.readFile ./tree-sitter-javascript.json)); + jsdoc = (builtins.fromJSON (builtins.readFile ./tree-sitter-jsdoc.json)); json = (builtins.fromJSON (builtins.readFile ./tree-sitter-json.json)); python = (builtins.fromJSON (builtins.readFile ./tree-sitter-python.json)); - # wasn’t able to check out with fetchgit - # ruby = (builtins.fromJSON (builtins.readFile ./tree-sitter-ruby.json)); + ruby = (builtins.fromJSON (builtins.readFile ./tree-sitter-ruby.json)); rust = (builtins.fromJSON (builtins.readFile ./tree-sitter-rust.json)); typescript = (builtins.fromJSON (builtins.readFile ./tree-sitter-typescript.json)); } diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-c.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-c.json index 1342f7d092b0..c44bd0344b3e 100644 --- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-c.json +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-c.json @@ -1,7 +1,7 @@ { "url": "https://github.com/tree-sitter/tree-sitter-c", - "rev": "22decdc361767838dd36f1da4125b35b5b9a3c28", - "date": "2019-07-02T15:49:42-07:00", - "sha256": "03f9g49l4g2l4hlafr3xhvi8d3a491k5zz4bxpq7391l5wgjy3zi", + "rev": "6002fcd5e86bb1e8670157bb008b97dbaf656d95", + "date": "2019-12-17T10:25:11-08:00", + "sha256": "0bcsgwyv9nskkh26hl4hylb03py98d2m565bgvq7fn3f8gyx5w7v", "fetchSubmodules": false } diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-cpp.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-cpp.json index 8c7d10f2654f..b17af74ac8c9 100644 --- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-cpp.json +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-cpp.json @@ -1,7 +1,7 @@ { "url": "https://github.com/tree-sitter/tree-sitter-cpp", - "rev": "f5afa0ee48ad1dc067ed6fe1aa2cfd2a3ea5d443", - "date": "2019-08-06T17:23:46-07:00", - "sha256": "1w9zjqj232fcagqfqd8qi4kmvr655s4ivllrm27973sda4xq557h", + "rev": "c7b767ac5fce26301321f91fb83fe9392c02d51d", + "date": "2019-12-17T13:43:08-08:00", + "sha256": "16i9iggr7ff18bif81viy9rvn1z55ad6yaq6dj0znjgaq8q6aclx", "fetchSubmodules": false } diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-embedded-template.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-embedded-template.json index 89940ffec8cb..a83a5a429e8b 100644 --- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-embedded-template.json +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-embedded-template.json @@ -1,7 +1,7 @@ { "url": "https://github.com/tree-sitter/tree-sitter-embedded-template", - "rev": "71955edec8cb762f63e94cf062fc96b52b9ae609", - "date": "2019-07-17T15:55:22-07:00", - "sha256": "1ar2n1z2h194lb3isbdkmvhn8w78j4a62nbh105w3jl1sxb4qpsa", + "rev": "3ca7c50fd3dd02df88cf120d8ab5b7b1f2efb270", + "date": "2020-01-16T16:31:20-08:00", + "sha256": "0vhmfn7jr5hdzksw808z99scqcyjr0i3qiybiq0yb644ysdmai05", "fetchSubmodules": false } diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-go.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-go.json index 6a354f5c2e23..bf8c4408b54c 100644 --- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-go.json +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-go.json @@ -1,7 +1,7 @@ { "url": "https://github.com/tree-sitter/tree-sitter-go", - "rev": "475571bb5bdb9b229c6be3843d4c71ba747688fd", - "date": "2019-07-17T15:51:06-07:00", - "sha256": "1cg5qpifrvpnsi0iy26g156xib2qa55vlna41hw6c70kx8ibvl9z", + "rev": "689cc8fbdc0613d267434f221af85aff91a31f8c", + "date": "2020-01-23T10:27:48-08:00", + "sha256": "04gpzi125akwgww5smw5x4dqr6jv5sq1165qn86bz15hwcmnybfj", "fetchSubmodules": false } diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-html.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-html.json index 732d2dda40b6..40d124b86269 100644 --- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-html.json +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-html.json @@ -1,7 +1,7 @@ { "url": "https://github.com/tree-sitter/tree-sitter-html", - "rev": "aeb2f456b8c6a60b8475d075889d476a165cde57", - "date": "2019-07-17T15:57:54-07:00", - "sha256": "0ba8zi65kja6p7f5h7pa7kxqa3mj29ysjrvl84am24vy5ik4zz3z", + "rev": "2147dd298a2b2e9593bfcb51a14c289a0dc7eb87", + "date": "2020-01-16T12:47:09-08:00", + "sha256": "0acg9v940xq4sw7019z2izc9qs10wwqipyiki243dws6z0wmwhdq", "fetchSubmodules": false } diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-javascript.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-javascript.json index a1e29e1504ea..04a6ae1056e8 100644 --- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-javascript.json +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-javascript.json @@ -1,7 +1,7 @@ { "url": "https://github.com/tree-sitter/tree-sitter-javascript", - "rev": "a730b5c210904e2e3c1f601125a059fde1b35850", - "date": "2019-08-08T14:13:17-07:00", - "sha256": "1cr0vikbzrklksjj07fh34a5cabkgbpkbxwiw2alnana3zzzdhnq", + "rev": "6e3692292adf8750fece625fc06fc1503e00d75e", + "date": "2020-01-27T13:47:50-08:00", + "sha256": "1i8m7v0859109bf2dz0dhf4cnhqmwc4947z49g4rs07k8h8dfk99", "fetchSubmodules": false } diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-jsdoc.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-jsdoc.json new file mode 100644 index 000000000000..9795f600c82b --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-jsdoc.json @@ -0,0 +1,7 @@ +{ + "url": "https://github.com/tree-sitter/tree-sitter-jsdoc", + "rev": "7b2dc910126d553cddd3b2318f39604d205b3a21", + "date": "2020-01-27T13:06:46-08:00", + "sha256": "04ahmbwfx3238dbsvlpg8xgb1c4bazdd8yw7dk4vph9cp8rq0dj5", + "fetchSubmodules": false +} diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-json.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-json.json index 2bb0dcd4d8f2..282e28c9be88 100644 --- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-json.json +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-json.json @@ -1,7 +1,7 @@ { "url": "https://github.com/tree-sitter/tree-sitter-json", - "rev": "337f55be9b9b1ccb0baa7763bfe014a94acea7ea", - "date": "2019-05-23T11:02:26-04:00", - "sha256": "0amh4qrjj3fli9c0z6p61z9d7496sqq54i1gh2vrghgnbbyaa6mz", + "rev": "7b6a33f300e3e88c3017e0a9d88c77b50ea6d149", + "date": "2019-12-10T10:27:22-08:00", + "sha256": "17yc7spn8v64ll684634zmjq3130yxvb5bfgjwg0agcaklpx71dp", "fetchSubmodules": false } diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-python.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-python.json index f46695edfa85..4abdc3fe00a2 100644 --- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-python.json +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-python.json @@ -1,7 +1,7 @@ { "url": "https://github.com/tree-sitter/tree-sitter-python", - "rev": "4c22de0944cd42a5c86ade7ef7097033604796f8", - "date": "2019-07-30T15:35:03-04:00", - "sha256": "1p12h7hj1ak15fyk4gw9wcmgzydd4z5mikhjp54mn1q4vfw175p3", + "rev": "899ac8d5d6c883b2f05362c9953e14e78aac474c", + "date": "2020-01-16T16:30:41-08:00", + "sha256": "1mw23kfr33s6md701p0p2nrd04119204ir78i7j1s5cxiyvpjq33", "fetchSubmodules": false } diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ruby.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ruby.json index e2f9928c8702..db58c7eb3eb9 100644 --- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ruby.json +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ruby.json @@ -1,7 +1,7 @@ { "url": "https://github.com/tree-sitter/tree-sitter-ruby", - "rev": "db91c934ff9d3d4ea67111a0f581532c49c3a6b3", - "date": "2019-07-26T15:51:54-06:00", - "sha256": "1ir1nqpz0c0hnsqzp90w2iw1gy3z3nqil2fm4n3zmid5di7c98dg", + "rev": "6540da2a79ceb7dc7afe50e9ac01eaad4378702d", + "date": "2019-12-09T14:56:14-08:00", + "sha256": "1i0ab1rm5hba0nnbmi4m37fbs368hk6wyby09pfmcmjwpi9zs0f9", "fetchSubmodules": false } diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-rust.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-rust.json index 328b337ed5a7..e3a906f076ac 100644 --- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-rust.json +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-rust.json @@ -1,7 +1,7 @@ { "url": "https://github.com/tree-sitter/tree-sitter-rust", - "rev": "3f956b18a6b0a576ed238cc69d5e3f413bd547b1", - "date": "2019-07-18T11:44:02-07:00", - "sha256": "0dwxg3pqirqm1lvl5x0q9djavfri9ffk5diygqzjnx53rwqhyzj8", + "rev": "1c37782a5528979a22991f8ed50dd3d3e423ac92", + "date": "2020-01-20T16:08:51-08:00", + "sha256": "04fzzn2rxpfg7pwymmarkdjabbp20yp10yzr4a5a1xcqx8hky9ix", "fetchSubmodules": false } diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-typescript.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-typescript.json index 61507888154d..9995665c7c68 100644 --- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-typescript.json +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-typescript.json @@ -1,7 +1,7 @@ { "url": "https://github.com/tree-sitter/tree-sitter-typescript", - "rev": "ab9ab6cced868ee3e096f33fa21fd9d356c92e1a", - "date": "2019-08-08T14:27:32-07:00", - "sha256": "11r0vj1dhv0my2cr442mwvaav8ljygsns20w51mwg7328vlz90q3", + "rev": "aa950f58ea8aa112bc72f3481b98fc2d3c07b3e0", + "date": "2020-01-27T13:51:28-08:00", + "sha256": "0y64lls7hw7yc70d9i6p4n1kchg11sbgl2m6yafy7w2ymksb5iw5", "fetchSubmodules": false } diff --git a/pkgs/development/tools/parsing/tree-sitter/update.nix b/pkgs/development/tools/parsing/tree-sitter/update.nix index aa87df6bed10..2a3575a44d20 100644 --- a/pkgs/development/tools/parsing/tree-sitter/update.nix +++ b/pkgs/development/tools/parsing/tree-sitter/update.nix @@ -22,7 +22,7 @@ let res=$(${curl}/bin/curl \ --silent \ "https://api.github.com/repos/${urlEscape owner}/$(${urlEscapeSh} "$repo")/releases/latest") - if [[ "$(printf "%s" "$res" | ${jq}bin/jq '.message')" =~ "rate limit" ]]; then + if [[ "$(printf "%s" "$res" | ${jq}/bin/jq '.message')" =~ "rate limit" ]]; then echo "rate limited" >&2 fi release=$(printf "%s" "$res" | ${jq}/bin/jq '.tag_name') diff --git a/pkgs/tools/text/ripgrep/default.nix b/pkgs/tools/text/ripgrep/default.nix index 0c72467c00b2..4a409b8dd84d 100644 --- a/pkgs/tools/text/ripgrep/default.nix +++ b/pkgs/tools/text/ripgrep/default.nix @@ -14,10 +14,7 @@ rustPlatform.buildRustPackage rec { sha256 = "1iga3320mgi7m853la55xip514a3chqsdi1a1rwv25lr9b1p7vd3"; }; - # Delete this on next update; see #79975 for details - legacyCargoFetcher = true; - - cargoSha256 = "17ldqr3asrdcsh4l29m3b5r37r5d0b3npq1lrgjmxb6vlx6a36qh"; + cargoSha256 = "0lwz661rbm7kwkd6mallxym1pz8ynda5f03ynjfd16vrazy2dj21"; cargoBuildFlags = stdenv.lib.optional withPCRE2 "--features pcre2";