nixos-render-docs: add options asciidoc converter

same reasoning as for the earlier commonmark converter.
This commit is contained in:
pennae 2023-02-05 05:58:14 +01:00 committed by pennae
parent 4d3aef762f
commit 417dd2ad16
5 changed files with 489 additions and 87 deletions

View file

@ -91,11 +91,14 @@ let
in rec {
inherit optionsNix;
optionsAsciiDoc = pkgs.runCommand "options.adoc" {} ''
${pkgs.python3Minimal}/bin/python ${./generateDoc.py} \
--format asciidoc \
optionsAsciiDoc = pkgs.runCommand "options.adoc" {
nativeBuildInputs = [ pkgs.nixos-render-docs ];
} ''
nixos-render-docs -j $NIX_BUILD_CORES options asciidoc \
--manpage-urls ${pkgs.path + "/doc/manpage-urls.json"} \
--revision ${lib.escapeShellArg revision} \
${optionsJSON}/share/doc/nixos/options.json \
> $out
$out
'';
optionsCommonMark = pkgs.runCommand "options.md" {

View file

@ -1,83 +0,0 @@
import argparse
import json
import sys
formats = ['asciidoc']
parser = argparse.ArgumentParser(
description = 'Generate documentation for a set of JSON-formatted NixOS options'
)
parser.add_argument(
'nix_options_path',
help = 'a path to a JSON file containing the NixOS options'
)
parser.add_argument(
'-f',
'--format',
choices = formats,
required = True,
help = f'the documentation format to generate'
)
args = parser.parse_args()
class OptionsEncoder(json.JSONEncoder):
def encode(self, obj):
# Unpack literal expressions and other Nix types.
# Don't escape the strings: they were escaped when initially serialized to JSON.
if isinstance(obj, dict):
_type = obj.get('_type')
if _type is not None:
if _type == 'literalExpression' or _type == 'literalDocBook':
return obj['text']
if _type == 'derivation':
return obj['name']
raise Exception(f'Unexpected type `{_type}` in {json.dumps(obj)}')
return super().encode(obj)
# TODO: declarations: link to github
def generate_asciidoc(options):
for (name, value) in options.items():
print(f'== {name}')
print()
print(value['description'])
print()
print('[discrete]')
print('=== details')
print()
print(f'Type:: {value["type"]}')
if 'default' in value:
print('Default::')
print('+')
print('----')
print(json.dumps(value['default'], cls=OptionsEncoder, ensure_ascii=False, separators=(',', ':')))
print('----')
print()
else:
print('No Default:: {blank}')
if value['readOnly']:
print('Read Only:: {blank}')
else:
print()
if 'example' in value:
print('Example::')
print('+')
print('----')
print(json.dumps(value['example'], cls=OptionsEncoder, ensure_ascii=False, separators=(',', ':')))
print('----')
print()
else:
print('No Example:: {blank}')
print()
with open(args.nix_options_path) as nix_options_json:
options = json.load(nix_options_json)
if args.format == 'asciidoc':
generate_asciidoc(options)
else:
raise Exception(f'Unsupported documentation format `--format {args.format}`')

View file

@ -0,0 +1,262 @@
from collections.abc import Mapping, MutableMapping, Sequence
from dataclasses import dataclass
from typing import Any, cast, Optional
from urllib.parse import quote
from .md import Renderer
import markdown_it
from markdown_it.token import Token
from markdown_it.utils import OptionsDict
_asciidoc_escapes = {
# escape all dots, just in case one is pasted at SOL
ord('.'): "{zwsp}.",
# may be replaced by typographic variants
ord("'"): "{apos}",
ord('"'): "{quot}",
# passthrough character
ord('+'): "{plus}",
# table marker
ord('|'): "{vbar}",
# xml entity reference
ord('&'): "{amp}",
# crossrefs. < needs extra escaping because links break in odd ways if they start with it
ord('<'): "{zwsp}+<+{zwsp}",
ord('>'): "{gt}",
# anchors, links, block attributes
ord('['): "{startsb}",
ord(']'): "{endsb}",
# superscript, subscript
ord('^'): "{caret}",
ord('~'): "{tilde}",
# bold
ord('*'): "{asterisk}",
# backslash
ord('\\'): "{backslash}",
# inline code
ord('`'): "{backtick}",
}
def asciidoc_escape(s: str) -> str:
s = s.translate(_asciidoc_escapes)
# :: is deflist item, ;; is has a replacement but no idea why
return s.replace("::", "{two-colons}").replace(";;", "{two-semicolons}")
@dataclass(kw_only=True)
class List:
head: str
@dataclass()
class Par:
sep: str
block_delim: str
continuing: bool = False
class AsciiDocRenderer(Renderer):
__output__ = "asciidoc"
_parstack: list[Par]
_list_stack: list[List]
_attrspans: list[str]
def __init__(self, manpage_urls: Mapping[str, str], parser: Optional[markdown_it.MarkdownIt] = None):
super().__init__(manpage_urls, parser)
self._parstack = [ Par("\n\n", "====") ]
self._list_stack = []
self._attrspans = []
def _enter_block(self, is_list: bool) -> None:
self._parstack.append(Par("\n+\n" if is_list else "\n\n", self._parstack[-1].block_delim + "="))
def _leave_block(self) -> None:
self._parstack.pop()
def _break(self, force: bool = False) -> str:
result = self._parstack[-1].sep if force or self._parstack[-1].continuing else ""
self._parstack[-1].continuing = True
return result
def _admonition_open(self, kind: str) -> str:
pbreak = self._break()
self._enter_block(False)
return f"{pbreak}[{kind}]\n{self._parstack[-2].block_delim}\n"
def _admonition_close(self) -> str:
self._leave_block()
return f"\n{self._parstack[-1].block_delim}\n"
def _list_open(self, token: Token, head: str) -> str:
attrs = []
if (idx := token.attrs.get('start')) is not None:
attrs.append(f"start={idx}")
if token.meta['compact']:
attrs.append('options="compact"')
if self._list_stack:
head *= len(self._list_stack[0].head) + 1
self._list_stack.append(List(head=head))
return f"{self._break()}[{','.join(attrs)}]"
def _list_close(self) -> str:
self._list_stack.pop()
return ""
def text(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
self._parstack[-1].continuing = True
return asciidoc_escape(token.content)
def paragraph_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._break()
def paragraph_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return ""
def hardbreak(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return " +\n"
def softbreak(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return f" "
def code_inline(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
self._parstack[-1].continuing = True
return f"``{asciidoc_escape(token.content)}``"
def code_block(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self.fence(token, tokens, i, options, env)
def link_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
self._parstack[-1].continuing = True
return f"link:{quote(cast(str, token.attrs['href']), safe='/:')}["
def link_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return "]"
def list_item_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
self._enter_block(True)
# allow the next token to be a block or an inline.
return f'\n{self._list_stack[-1].head} {{empty}}'
def list_item_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
self._leave_block()
return "\n"
def bullet_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._list_open(token, '*')
def bullet_list_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._list_close()
def em_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return "__"
def em_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return "__"
def strong_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return "**"
def strong_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return "**"
def fence(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
attrs = f"[source,{token.info}]\n" if token.info else ""
code = token.content
if code.endswith('\n'):
code = code[:-1]
return f"{self._break(True)}{attrs}----\n{code}\n----"
def blockquote_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
pbreak = self._break(True)
self._enter_block(False)
return f"{pbreak}[quote]\n{self._parstack[-2].block_delim}\n"
def blockquote_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
self._leave_block()
return f"\n{self._parstack[-1].block_delim}"
def note_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._admonition_open("NOTE")
def note_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._admonition_close()
def caution_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._admonition_open("CAUTION")
def caution_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._admonition_close()
def important_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._admonition_open("IMPORTANT")
def important_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._admonition_close()
def tip_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._admonition_open("TIP")
def tip_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._admonition_close()
def warning_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._admonition_open("WARNING")
def warning_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._admonition_close()
def dl_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return f"{self._break()}[]"
def dl_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return ""
def dt_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._break()
def dt_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
self._enter_block(True)
return ":: {empty}"
def dd_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return ""
def dd_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
self._leave_block()
return "\n"
def myst_role(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
self._parstack[-1].continuing = True
content = asciidoc_escape(token.content)
if token.meta['name'] == 'manpage' and (url := self._manpage_urls.get(token.content)):
return f"link:{quote(url, safe='/:')}[{content}]"
return f"[.{token.meta['name']}]``{asciidoc_escape(token.content)}``"
def inline_anchor(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
self._parstack[-1].continuing = True
return f"[[{token.attrs['id']}]]"
def attr_span_begin(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
self._parstack[-1].continuing = True
(id_part, class_part) = ("", "")
if id := token.attrs.get('id'):
id_part = f"[[{id}]]"
if s := token.attrs.get('class'):
if s == 'keycap':
class_part = "kbd:["
self._attrspans.append("]")
else:
return super().attr_span_begin(token, tokens, i, options, env)
else:
self._attrspans.append("")
return id_part + class_part
def attr_span_end(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._attrspans.pop()
def heading_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return token.markup.replace("#", "=") + " "
def heading_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return "\n"
def ordered_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._list_open(token, '.')
def ordered_list_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._list_close()

View file

@ -8,11 +8,13 @@ from collections.abc import Mapping, MutableMapping, Sequence
from markdown_it.utils import OptionsDict
from markdown_it.token import Token
from typing import Any, Optional
from urllib.parse import quote
from xml.sax.saxutils import escape, quoteattr
import markdown_it
from . import parallel
from .asciidoc import AsciiDocRenderer, asciidoc_escape
from .commonmark import CommonMarkRenderer
from .docbook import DocBookRenderer, make_xml_id
from .manpage import ManpageRenderer, man_escape
@ -476,6 +478,59 @@ class CommonMarkConverter(BaseConverter):
return "\n".join(result)
class OptionsAsciiDocRenderer(OptionDocsRestrictions, AsciiDocRenderer):
pass
class AsciiDocConverter(BaseConverter):
__renderer__ = AsciiDocRenderer
__option_block_separator__ = ""
def _parallel_render_prepare(self) -> Any:
return (self._manpage_urls, self._revision, self._markdown_by_default)
@classmethod
def _parallel_render_init_worker(cls, a: Any) -> AsciiDocConverter:
return cls(*a)
def _render_code(self, option: dict[str, Any], key: str) -> list[str]:
# NOTE this duplicates the old direct-paste behavior, even if it is somewhat
# incorrect, since users rely on it.
if lit := option_is(option, key, 'literalDocBook'):
return [ f"*{key.capitalize()}:* {lit['text']}" ]
else:
return super()._render_code(option, key)
def _render_description(self, desc: str | dict[str, Any]) -> list[str]:
# NOTE this duplicates the old direct-paste behavior, even if it is somewhat
# incorrect, since users rely on it.
if isinstance(desc, str) and not self._markdown_by_default:
return [ desc ]
else:
return super()._render_description(desc)
def _related_packages_header(self) -> list[str]:
return [ "__Related packages:__" ]
def _decl_def_header(self, header: str) -> list[str]:
return [ f"__{header}:__\n" ]
def _decl_def_entry(self, href: Optional[str], name: str) -> list[str]:
if href is not None:
return [ f"* link:{quote(href, safe='/:')}[{asciidoc_escape(name)}]" ]
return [ f"* {asciidoc_escape(name)}" ]
def _decl_def_footer(self) -> list[str]:
return []
def finalize(self) -> str:
result = []
for (name, opt) in self._sorted_options():
result.append(f"== {asciidoc_escape(name)}\n")
result += opt.lines
result.append("\n\n")
return "\n".join(result)
def _build_cli_db(p: argparse.ArgumentParser) -> None:
p.add_argument('--manpage-urls', required=True)
p.add_argument('--revision', required=True)
@ -498,6 +553,13 @@ def _build_cli_commonmark(p: argparse.ArgumentParser) -> None:
p.add_argument("infile")
p.add_argument("outfile")
def _build_cli_asciidoc(p: argparse.ArgumentParser) -> None:
p.add_argument('--manpage-urls', required=True)
p.add_argument('--revision', required=True)
p.add_argument('--markdown-by-default', default=False, action='store_true')
p.add_argument("infile")
p.add_argument("outfile")
def _run_cli_db(args: argparse.Namespace) -> None:
with open(args.manpage_urls, 'r') as manpage_urls:
md = DocBookConverter(
@ -537,11 +599,24 @@ def _run_cli_commonmark(args: argparse.Namespace) -> None:
with open(args.outfile, 'w') as f:
f.write(md.finalize())
def _run_cli_asciidoc(args: argparse.Namespace) -> None:
with open(args.manpage_urls, 'r') as manpage_urls:
md = AsciiDocConverter(
json.load(manpage_urls),
revision = args.revision,
markdown_by_default = args.markdown_by_default)
with open(args.infile, 'r') as f:
md.add_options(json.load(f))
with open(args.outfile, 'w') as f:
f.write(md.finalize())
def build_cli(p: argparse.ArgumentParser) -> None:
formats = p.add_subparsers(dest='format', required=True)
_build_cli_db(formats.add_parser('docbook'))
_build_cli_manpage(formats.add_parser('manpage'))
_build_cli_commonmark(formats.add_parser('commonmark'))
_build_cli_asciidoc(formats.add_parser('asciidoc'))
def run_cli(args: argparse.Namespace) -> None:
if args.format == 'docbook':
@ -550,5 +625,7 @@ def run_cli(args: argparse.Namespace) -> None:
_run_cli_manpage(args)
elif args.format == 'commonmark':
_run_cli_commonmark(args)
elif args.format == 'asciidoc':
_run_cli_asciidoc(args)
else:
raise RuntimeError('format not hooked up', args)

View file

@ -0,0 +1,143 @@
import nixos_render_docs
from sample_md import sample1
class Converter(nixos_render_docs.md.Converter):
__renderer__ = nixos_render_docs.asciidoc.AsciiDocRenderer
def test_lists() -> None:
c = Converter({})
# attaching to the nth ancestor list requires n newlines before the +
assert c._render("""\
- a
b
- c
- d
- e
1
f
""") == """\
[]
* {empty}a
+
b
* {empty}c
+
[options="compact"]
** {empty}d
+
[]
** {empty}e
+
1
+
f
"""
def test_full() -> None:
c = Converter({ 'man(1)': 'http://example.org' })
assert c._render(sample1) == """\
[WARNING]
====
foo
[NOTE]
=====
nested
=====
====
link:link[ multiline ]
link:http://example.org[man(1)] reference
[[b]]some [[a]]nested anchors
__emph__ **strong** __nesting emph **and strong** and ``code``__
[]
* {empty}wide bullet
* {empty}list
[]
. {empty}wide ordered
. {empty}list
[options="compact"]
* {empty}narrow bullet
* {empty}list
[options="compact"]
. {empty}narrow ordered
. {empty}list
[quote]
====
quotes
[quote]
=====
with __nesting__
----
nested code block
----
=====
[options="compact"]
* {empty}and lists
* {empty}
+
----
containing code
----
and more quote
====
[start=100,options="compact"]
. {empty}list starting at 100
. {empty}goes on
[]
deflist:: {empty}
+
[quote]
=====
with a quote and stuff
=====
+
----
code block
----
+
----
fenced block
----
+
text
more stuff in same deflist:: {empty}foo
"""