repl: improve :doc builtin repl command to support lambdas.

For a long time `nix repl` has supported displaying documentation set on
builtins, however, it has long been convention to use Markdown comments
on Nix functions themselves for documentation. This exposes that
information to `nix repl` users in a nice and formatted way.

NixOS/rfcs#145 doc-comments are primarily what this feature is intended
to consume, however, support for lambda documentation in the repl is
experimental. We do our best effort to support the RFC here.

These changes are based on [the nix-doc library](https://github.com/lf-/nix-doc) and
are licensed under the terms described in the relevant source files.

Change-Id: Ic6fe947d39a22540705d890737e336c4720b0a22
This commit is contained in:
Lunaphied 2024-03-17 18:01:05 -06:00
parent 56c7dfd652
commit 7a1054fa5f
15 changed files with 626 additions and 5 deletions

View file

@ -19,6 +19,7 @@ LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
LIBCURL_LIBS = @LIBCURL_LIBS@
LIBSECCOMP_LIBS = @LIBSECCOMP_LIBS@
LOWDOWN_LIBS = @LOWDOWN_LIBS@
NIXDOC_LIBS = -lnix_doc
OPENSSL_LIBS = @OPENSSL_LIBS@
PACKAGE_NAME = @PACKAGE_NAME@
PACKAGE_VERSION = @PACKAGE_VERSION@

View file

@ -0,0 +1,13 @@
---
synopsis: Experimental REPL support for documentation comments using `:doc`
cls: 564
---
Using `:doc` in the REPL now supports showing documentation comments when defined on a function.
Previously this was only able to document builtins, however it now will show comments defined on a lambda as well.
This support is experimental and relies on an embedded version of [nix-doc](https://github.com/lf-/nix-doc).
The logic also supports limited Markdown formatting of doccomments and should easily support any [RFC 145](https://github.com/NixOS/rfcs/blob/master/rfcs/0145-doc-strings.md)
compatible documentation comments in addition to simple commented documentation.

View file

@ -198,11 +198,14 @@
'';
};
nix-doc = final.callPackage ./nix-doc/package.nix {};
nix = final.callPackage ./package.nix {
inherit versionSuffix fileset;
stdenv = currentStdenv;
boehmgc = final.boehmgc-nix;
busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell;
nix-doc = final.nix-doc;
};
};

View file

@ -38,7 +38,7 @@ for p in paths:
try:
e = frontmatter.load(p)
if 'synopsis' not in e.metadata:
raise Exception('missing synposis')
raise Exception('missing synopsis')
unknownKeys = set(e.metadata.keys()) - set(('synopsis', 'cls', 'issues', 'prs', 'significance'))
if unknownKeys:
raise Exception('unknown keys', unknownKeys)

View file

@ -219,6 +219,11 @@ deps += toml11
nlohmann_json = dependency('nlohmann_json', required : true)
deps += nlohmann_json
# nix-doc is a Rust project provided via buildInputs and unfortunately doesn't have any way to be detected.
# Just declare it manually to resolve this.
nix_doc = declare_dependency(link_args : [ '-lnix_doc' ])
deps += nix_doc
#
# Build-time tools
#

6
nix-doc/.gitignore vendored Normal file
View file

@ -0,0 +1,6 @@
# SPDX-FileCopyrightText: 2024 Jade Lovelace
#
# SPDX-License-Identifier: BSD-2-Clause OR MIT
/target
result

161
nix-doc/Cargo.lock generated Normal file
View file

@ -0,0 +1,161 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 3
[[package]]
name = "autocfg"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "cbitset"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29b6ad25ae296159fb0da12b970b2fe179b234584d7cd294c891e2bbb284466b"
dependencies = [
"num-traits",
]
[[package]]
name = "dissimilar"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86e3bdc80eee6e16b2b6b0f87fbc98c04bee3455e35174c0de1a125d0688c632"
[[package]]
name = "expect-test"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30d9eafeadd538e68fb28016364c9732d78e420b9ff8853fa5e4058861e9f8d3"
dependencies = [
"dissimilar",
"once_cell",
]
[[package]]
name = "nix-doc"
version = "0.0.1"
dependencies = [
"expect-test",
"rnix",
]
[[package]]
name = "num-traits"
version = "0.2.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a"
dependencies = [
"autocfg",
]
[[package]]
name = "once_cell"
version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
[[package]]
name = "proc-macro2"
version = "1.0.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef"
dependencies = [
"proc-macro2",
]
[[package]]
name = "rnix"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a9b645f0edba447dbfc6473dd22999f46a1d00ab39e777a2713a1cf34a1597b"
dependencies = [
"cbitset",
"rowan",
]
[[package]]
name = "rowan"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ea7cadf87a9d8432e85cb4eb86bd2e765ace60c24ef86e79084dcae5d1c5a19"
dependencies = [
"rustc-hash",
"smol_str",
"text_unit",
"thin-dst",
]
[[package]]
name = "rustc-hash"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
[[package]]
name = "serde"
version = "1.0.197"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.197"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "smol_str"
version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fad6c857cbab2627dcf01ec85a623ca4e7dcb5691cbaa3d7fb7653671f0d09c9"
dependencies = [
"serde",
]
[[package]]
name = "syn"
version = "2.0.53"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7383cd0e49fff4b6b90ca5670bfd3e9d6a733b3f90c686605aa7eec8c4996032"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "text_unit"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20431e104bfecc1a40872578dbc390e10290a0e9c35fffe3ce6f73c15a9dbfc2"
[[package]]
name = "thin-dst"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db3c46be180f1af9673ebb27bc1235396f61ef6965b3fe0dbb2e624deb604f0e"
[[package]]
name = "unicode-ident"
version = "1.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"

17
nix-doc/Cargo.toml Normal file
View file

@ -0,0 +1,17 @@
[package]
description = "Nix documentation grepping tool"
edition = "2018"
name = "nix-doc"
version = "0.0.1"
license = "BSD-2-Clause OR MIT"
homepage = "https://github.com/lf-/nix-doc"
repository = "https://github.com/lf-/nix-doc"
[lib]
crate_type = ["staticlib"]
[dependencies]
rnix = "0.8.0"
[dev-dependencies]
expect-test = "1.1.0"

11
nix-doc/package.nix Normal file
View file

@ -0,0 +1,11 @@
{
rustPlatform,
lib
}:
rustPlatform.buildRustPackage {
name = "nix-doc";
cargoHash = "sha256-HXL235loBJnRje7KaMCCCTighv6WNYRrZ/jgkAQbEY0=";
src = lib.cleanSource ./.;
}

326
nix-doc/src/lib.rs Normal file
View file

@ -0,0 +1,326 @@
// SPDX-FileCopyrightText: 2024 Jade Lovelace
//
// SPDX-License-Identifier: BSD-2-Clause OR MIT
//! library components of nix-doc
pub mod pprint;
use crate::pprint::pprint_args;
use rnix::types::{Lambda, TypedNode};
use rnix::SyntaxKind::*;
use rnix::{NodeOrToken, SyntaxNode, TextUnit, WalkEvent};
use std::ffi::{CStr, CString};
use std::fs;
use std::iter;
use std::os::raw::c_char;
use std::panic;
use std::ptr;
use std::{fmt::Display, str};
pub type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
const DOC_INDENT: usize = 3;
struct SearchResult {
/// Name of the function
identifier: String,
/// Dedented documentation comments
doc: String,
/// Parameter block for the function
param_block: String,
}
fn find_pos(file: &str, line: usize, col: usize) -> usize {
let mut lines = 1;
let mut line_start = 0;
let mut it = file.chars().enumerate().peekable();
while let Some((count, ch)) = it.next() {
if ch == '\n' || ch == '\r' {
lines += 1;
let addend = if ch == '\r' && it.peek().map(|x| x.1) == Some('\n') {
it.next();
1
} else {
0
};
line_start = count + addend;
}
let col_diff = ((count as i32) - (line_start as i32)).abs() as usize;
if lines == line && col_diff == col {
return count;
}
}
unreachable!();
}
impl SearchResult {
fn format<P: Display>(&self, filename: P, line: usize) -> String {
format!(
"**Synopsis:** `{}` = {}\n\n{}\n\n# {}",
self.identifier.as_str(),
self.param_block,
indented(&self.doc, DOC_INDENT),
format!("{}:{}", filename, line).as_str(),
)
}
}
/// Emits a string `s` indented by `indent` spaces
fn indented(s: &str, indent: usize) -> String {
let indent_s = iter::repeat(' ').take(indent).collect::<String>();
s.split('\n')
.map(|line| indent_s.clone() + line)
.collect::<Vec<_>>()
.join("\n")
}
/// Cleans up a single line, erasing prefix single line comments but preserving indentation
fn cleanup_single_line<'a>(s: &'a str) -> &'a str {
let mut cmt_new_start = 0;
for (idx, ch) in s.char_indices() {
// if we find a character, save the byte position after it as our new string start
if ch == '#' || ch == '*' {
cmt_new_start = idx + 1;
break;
}
// if, instead, we are on a line with no starting comment characters, leave it alone as it
// will be handled by dedent later
if !ch.is_whitespace() {
break;
}
}
&s[cmt_new_start..]
}
/// Erases indents in comments. This is *almost* a normal dedent function, but it starts by looking
/// at the second line if it can.
fn dedent_comment(s: &str) -> String {
let mut whitespaces = 0;
let mut lines = s.lines();
let first = lines.next();
// scan for whitespace
for line in lines.chain(first) {
let line_whitespace = line.chars().take_while(|ch| ch.is_whitespace()).count();
if line_whitespace != line.len() {
// a non-whitespace line, perfect for taking whitespace off of
whitespaces = line_whitespace;
break;
}
}
// maybe the first considered line we found was indented further, so let's look for more lines
// that might have a shorter indent. In the case of one line, do nothing.
for line in s.lines().skip(1) {
let line_whitespace = line.chars().take_while(|ch| ch.is_whitespace()).count();
if line_whitespace != line.len() {
whitespaces = line_whitespace.min(whitespaces);
}
}
// delete up to `whitespaces` whitespace characters from each line and reconstitute the string
let mut out = String::new();
for line in s.lines() {
let content_begin = line.find(|ch: char| !ch.is_whitespace()).unwrap_or(0);
out.push_str(&line[content_begin.min(whitespaces)..]);
out.push('\n');
}
out.truncate(out.trim_end_matches('\n').len());
out
}
/// Deletes whitespace and leading comment characters
///
/// Oversight we are choosing to ignore: if you put # characters at the beginning of lines in a
/// multiline comment, they will be deleted.
fn cleanup_comments<S: AsRef<str>, I: DoubleEndedIterator<Item = S>>(comment: &mut I) -> String {
dedent_comment(
&comment
.rev()
.map(|small_comment| {
small_comment
.as_ref()
// space before multiline start
.trim_start()
// multiline starts
.trim_start_matches("/*")
// trailing so we can grab multiline end
.trim_end()
// multiline ends
.trim_end_matches("*/")
// extra space that was in the multiline
.trim()
.split('\n')
// erase single line comments and such
.map(cleanup_single_line)
.collect::<Vec<_>>()
.join("\n")
})
.collect::<Vec<_>>()
.join("\n"),
)
}
/// Get the docs for a specific function
pub fn get_function_docs(filename: &str, line: usize, col: usize) -> Option<String> {
let content = fs::read(filename).ok()?;
let decoded = str::from_utf8(&content).ok()?;
let pos = find_pos(&decoded, line, col);
let rowan_pos = TextUnit::from_usize(pos);
let tree = rnix::parse(decoded);
let mut lambda = None;
for node in tree.node().preorder() {
match node {
WalkEvent::Enter(n) => {
if n.text_range().start() >= rowan_pos && n.kind() == NODE_LAMBDA {
lambda = Lambda::cast(n);
break;
}
}
WalkEvent::Leave(_) => (),
}
}
let lambda = lambda?;
let res = visit_lambda("func".to_string(), &lambda);
Some(res.format(filename, line))
}
fn visit_lambda(name: String, lambda: &Lambda) -> SearchResult {
// grab the arguments
let param_block = pprint_args(&lambda);
// find the doc comment
let comment = find_comment(lambda.node().clone()).unwrap_or_else(|| "".to_string());
SearchResult {
identifier: name,
doc: comment,
param_block
}
}
fn find_comment(node: SyntaxNode) -> Option<String> {
let mut node = NodeOrToken::Node(node);
let mut comments = Vec::new();
loop {
loop {
if let Some(new) = node.prev_sibling_or_token() {
node = new;
break;
} else {
node = NodeOrToken::Node(node.parent()?);
}
}
match node.kind() {
TOKEN_COMMENT => match &node {
NodeOrToken::Token(token) => comments.push(token.text().clone()),
NodeOrToken::Node(_) => unreachable!(),
},
// This stuff is found as part of `the-fn = f: ...`
// here: ^^^^^^^^
NODE_KEY | TOKEN_ASSIGN => (),
t if t.is_trivia() => (),
_ => break,
}
}
let doc = cleanup_comments(&mut comments.iter().map(|c| c.as_str()));
Some(doc).filter(|it| !it.is_empty())
}
/// Get the docs for a function in the given file path at the given file position and return it as
/// a C string pointer
#[no_mangle]
pub extern "C" fn nd_get_function_docs(
filename: *const c_char,
line: usize,
col: usize,
) -> *const c_char {
let fname = unsafe { CStr::from_ptr(filename) };
fname
.to_str()
.ok()
.and_then(|f| {
panic::catch_unwind(|| get_function_docs(f, line, col))
.map_err(|e| {
eprintln!("panic!! {:#?}", e);
e
})
.ok()
})
.flatten()
.and_then(|s| CString::new(s).ok())
.map(|s| s.into_raw() as *const c_char)
.unwrap_or(ptr::null())
}
/// Call this to free a string from nd_get_function_docs
#[no_mangle]
pub extern "C" fn nd_free_string(s: *const c_char) {
unsafe {
// cast note: this cast is turning something that was cast to const
// back to mut
drop(CString::from_raw(s as *mut c_char));
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_bytepos() {
let fakefile = "abc\ndef\nghi";
assert_eq!(find_pos(fakefile, 2, 2), 5);
}
#[test]
fn test_bytepos_cursed() {
let fakefile = "abc\rdef\r\nghi";
assert_eq!(find_pos(fakefile, 2, 2), 5);
assert_eq!(find_pos(fakefile, 3, 2), 10);
}
#[test]
fn test_comment_stripping() {
let ex1 = ["/* blah blah blah\n foooo baaar\n blah */"];
assert_eq!(
cleanup_comments(&mut ex1.iter()),
"blah blah blah\n foooo baaar\nblah"
);
let ex2 = ["# a1", "# a2", "# aa"];
assert_eq!(cleanup_comments(&mut ex2.iter()), "aa\n a2\na1");
}
#[test]
fn test_dedent() {
let ex1 = "a\n b\n c\n d";
assert_eq!(dedent_comment(ex1), "a\nb\nc\n d");
let ex2 = "a\nb\nc";
assert_eq!(dedent_comment(ex2), ex2);
let ex3 = " a\n b\n\n c";
assert_eq!(dedent_comment(ex3), "a\nb\n\n c");
}
#[test]
fn test_single_line_comment_stripping() {
let ex1 = " * a";
let ex2 = " # a";
let ex3 = " a";
assert_eq!(cleanup_single_line(ex1), " a");
assert_eq!(cleanup_single_line(ex2), " a");
assert_eq!(cleanup_single_line(ex3), ex3);
}
}

40
nix-doc/src/pprint.rs Normal file
View file

@ -0,0 +1,40 @@
// SPDX-FileCopyrightText: 2024 Jade Lovelace
//
// SPDX-License-Identifier: BSD-2-Clause OR MIT
use rnix::types::{Lambda, TypedNode};
use rnix::SyntaxKind::*;
/// Pretty-prints the arguments to a function
pub fn pprint_args(lambda: &Lambda) -> String {
// TODO: handle docs directly on NODE_IDENT args (uncommon case)
let mut lambda = lambda.clone();
let mut out = String::new();
loop {
let arg = lambda.arg().unwrap();
match arg.kind() {
NODE_IDENT => {
out += &format!("*{}*", &arg.to_string());
out.push_str(": ");
let body = lambda.body().unwrap();
if body.kind() == NODE_LAMBDA {
lambda = Lambda::cast(body).unwrap();
} else {
break;
}
}
NODE_PATTERN => {
out += &format!("*{}*", &arg.to_string());
out.push_str(": ");
break;
}
t => {
unreachable!("unhandled arg type {:?}", t);
}
}
}
out.push_str("...");
out
//pprint_arg(lambda.arg());
}

View file

@ -43,6 +43,8 @@
busybox-sandbox-shell,
nix-doc,
pname ? "nix",
versionSuffix ? "",
officialRelease ? true,
@ -186,6 +188,7 @@ in stdenv.mkDerivation (finalAttrs: {
lowdown
libsodium
toml11
nix-doc
]
++ lib.optionals stdenv.hostPlatform.isLinux [ libseccomp busybox-sandbox-shell ]
++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid

View file

@ -8,7 +8,7 @@ libcmd_SOURCES := $(wildcard $(d)/*.cc)
libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers
libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread
libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) $(NIXDOC_LIBS) -pthread
libcmd_LIBS = libstore libutil libexpr libmain libfetchers

View file

@ -45,6 +45,7 @@ libcmd = library(
editline,
lowdown,
nlohmann_json,
nix_doc
],
install : true,
# FIXME(Qyriad): is this right?

View file

@ -36,8 +36,26 @@
#include <gc/gc_cpp.h>
#endif
// XXX: These are for nix-doc features and will be removed in a future rewrite where this functionality is integrated more natively.
extern "C" {
char const *nd_get_function_docs(char const *filename, size_t line, size_t col);
void nd_free_string(char const *str);
}
namespace nix {
/** Wrapper around std::unique_ptr with a custom deleter for strings from nix-doc **/
using NdString = std::unique_ptr<const char, decltype(&nd_free_string)>;
/**
* Fetch a string representing the doc comment using nix-doc and wrap it in an RAII wrapper.
*/
NdString lambdaDocsForPos(SourcePath const path, nix::Pos const &pos) {
std::string const file = path.to_string();
return NdString{nd_get_function_docs(file.c_str(), pos.line, pos.column), &nd_free_string};
}
/**
* Returned by `NixRepl::processLine`.
*/
@ -400,7 +418,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
<< " nix-shell\n"
<< " :t <expr> Describe result of evaluation\n"
<< " :u <expr> Build derivation, then start nix-shell\n"
<< " :doc <expr> Show documentation of a builtin function\n"
<< " :doc <expr> Show documentation for the provided function (experimental lambda support)\n"
<< " :log <expr> Show logs for a derivation\n"
<< " :te, :trace-enable [bool] Enable, disable or toggle showing traces for\n"
<< " errors\n"
@ -629,8 +647,24 @@ ProcessLineResult NixRepl::processLine(std::string line)
markdown += stripIndentation(doc->doc);
logger->cout(trim(renderMarkdownToTerminal(markdown)));
} else
throw Error("value does not have documentation");
} else if (v.isLambda()) {
auto pos = state->positions[v.lambda.fun->pos];
if (auto path = std::get_if<SourcePath>(&pos.origin)) {
// Path and position have now been obtained, feed to nix-doc library to get data.
auto docComment = lambdaDocsForPos(*path, pos);
if (!docComment) {
throw Error("lambda '%s' has no documentation comment", pos);
}
// Build and print Markdown representation of documentation comment.
std::string markdown = stripIndentation(docComment.get());
logger->cout(trim(renderMarkdownToTerminal(markdown)));
} else {
throw Error("lambda '%s' doesn't have a determinable source file", pos);
}
} else {
throw Error("value '%s' does not have documentation", arg);
}
}
else if (command == ":te" || command == ":trace-enable") {