From 2b35c4dccfd8b0596cff84f1e2b9dd1ceadd9580 Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Mon, 13 Jan 2025 15:19:10 +1300 Subject: [PATCH] Docs: Proto doc_string approach for cmd help Add `doc_string` field to `Pass` constructor Add `docs/util/newcmdref.py` to contain command domain Update `docs/util/cmdref.py` with `cmd:usage` and `cmd:optiongroup` for describing commands. Functional, but WIP. --- docs/source/conf.py | 2 + docs/util/cmdref.py | 131 ++++++++++++- docs/util/newcmdref.py | 427 +++++++++++++++++++++++++++++++++++++++++ kernel/register.cc | 112 +++++++---- kernel/register.h | 6 + 5 files changed, 635 insertions(+), 43 deletions(-) create mode 100644 docs/util/newcmdref.py diff --git a/docs/source/conf.py b/docs/source/conf.py index 8c8555b71..e544d938f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -84,6 +84,8 @@ extensions.append('util.cmdref') extensions.append('sphinx.ext.autodoc') extensions.append('util.cellref') cells_json = Path(__file__).parent / 'generated' / 'cells.json' +extensions.append('util.newcmdref') +cmds_json = Path(__file__).parent / 'generated' / 'cmds.json' from sphinx.application import Sphinx def setup(app: Sphinx) -> None: diff --git a/docs/util/cmdref.py b/docs/util/cmdref.py index a31b08e0d..203d9369e 100644 --- a/docs/util/cmdref.py +++ b/docs/util/cmdref.py @@ -4,10 +4,12 @@ from __future__ import annotations import re from typing import cast +import warnings from docutils import nodes -from docutils.nodes import Node, Element, system_message +from docutils.nodes import Node, Element from docutils.parsers.rst import directives +from docutils.parsers.rst.roles import GenericRole from docutils.parsers.rst.states import Inliner from sphinx.application import Sphinx from sphinx.domains import Domain, Index @@ -17,7 +19,7 @@ from sphinx.roles import XRefRole from sphinx.directives import ObjectDescription from sphinx.directives.code import container_wrapper from sphinx.util.nodes import make_refnode -from sphinx.util.docfields import Field +from sphinx.util.docfields import Field, GroupedField from sphinx import addnodes class TocNode(ObjectDescription): @@ -63,10 +65,15 @@ class CommandNode(TocNode): name = 'cmd' required_arguments = 1 - option_spec = { + option_spec = TocNode.option_spec.copy() + option_spec.update({ 'title': directives.unchanged, 'tags': directives.unchanged - } + }) + + doc_field_types = [ + GroupedField('opts', label='Options', names=('option', 'options', 'opt', 'opts')), + ] def handle_signature(self, sig, signode: addnodes.desc_signature): signode['fullname'] = sig @@ -93,6 +100,120 @@ class CommandNode(TocNode): idx, 0)) +class CommandUsageNode(TocNode): + """A custom node that describes command usages""" + + name = 'cmdusage' + + option_spec = TocNode.option_spec + option_spec.update({ + 'usage': directives.unchanged, + }) + + doc_field_types = [ + GroupedField('opts', label='Options', names=('option', 'options', 'opt', 'opts')), + ] + + def handle_signature(self, sig: str, signode: addnodes.desc_signature): + try: + cmd, use = sig.split('::') + except ValueError: + cmd, use = sig, '' + signode['fullname'] = sig + usage = self.options.get('usage', use or sig) + if usage: + signode['tocname'] = usage + signode += addnodes.desc_name(text=usage) + return signode['fullname'] + + def add_target_and_index( + self, + name: str, + sig: str, + signode: addnodes.desc_signature + ) -> None: + idx = ".".join(name.split("::")) + signode['ids'].append(idx) + if 'noindex' not in self.options: + tocname: str = signode.get('tocname', name) + objs = self.env.domaindata[self.domain]['objects'] + # (name, sig, typ, docname, anchor, prio) + objs.append((name, + tocname, + type(self).name, + self.env.docname, + idx, + 1)) + +class CommandOptionGroupNode(TocNode): + """A custom node that describes a group of related options""" + + name = 'cmdoptiongroup' + + option_spec = TocNode.option_spec + + doc_field_types = [ + Field('opt', ('option',), label='', rolename='option') + ] + + def handle_signature(self, sig: str, signode: addnodes.desc_signature): + try: + cmd, name = sig.split('::') + except ValueError: + cmd, name = '', sig + signode['fullname'] = sig + signode['tocname'] = name + signode += addnodes.desc_name(text=name) + return signode['fullname'] + + def add_target_and_index( + self, + name: str, + sig: str, + signode: addnodes.desc_signature + ) -> None: + idx = ".".join(name.split("::")) + signode['ids'].append(idx) + if 'noindex' not in self.options: + tocname: str = signode.get('tocname', name) + objs = self.env.domaindata[self.domain]['objects'] + # (name, sig, typ, docname, anchor, prio) + objs.append((name, + tocname, + type(self).name, + self.env.docname, + idx, + 1)) + + def transform_content(self, contentnode: addnodes.desc_content) -> None: + """hack `:option -thing: desc` into a proper option list""" + newchildren = [] + for node in contentnode: + newnode = node + if isinstance(node, nodes.field_list): + newnode = nodes.option_list() + for field in node: + is_option = False + option_list_item = nodes.option_list_item() + for child in field: + if isinstance(child, nodes.field_name): + option_group = nodes.option_group() + option_list_item += option_group + option = nodes.option() + option_group += option + name, text = child.rawsource.split(' ', 1) + is_option = name == 'option' + option += nodes.option_string(text=text) + if not is_option: warnings.warn(f'unexpected option \'{name}\' in {field.source}') + elif isinstance(child, nodes.field_body): + description = nodes.description() + description += child.children + option_list_item += description + if is_option: + newnode += option_list_item + newchildren.append(newnode) + contentnode.children = newchildren + class PropNode(TocNode): name = 'prop' fieldname = 'props' @@ -517,6 +638,8 @@ class CommandDomain(Domain): directives = { 'def': CommandNode, + 'usage': CommandUsageNode, + 'optiongroup': CommandOptionGroupNode, } indices = { diff --git a/docs/util/newcmdref.py b/docs/util/newcmdref.py new file mode 100644 index 000000000..4b5995065 --- /dev/null +++ b/docs/util/newcmdref.py @@ -0,0 +1,427 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +from dataclasses import dataclass +import json +from pathlib import Path, PosixPath, WindowsPath +import re + +from typing import Any +from sphinx.application import Sphinx +from sphinx.ext import autodoc +from sphinx.ext.autodoc import Documenter +from sphinx.util import logging + +logger = logging.getLogger(__name__) + +# cmd signature +cmd_ext_sig_re = re.compile( + r'''^ ([\w$._]+?) # module name + (?:\.([\w_]+))? # optional: thing name + (::[\w_]+)? # attribute + \s* $ # and nothing more + ''', re.VERBOSE) + +@dataclass +class YosysCmdUsage: + signature: str + description: str + options: list[tuple[str,str]] + postscript: str + +class YosysCmd: + name: str + title: str + content: list[str] + usages: list[YosysCmdUsage] + experimental_flag: bool + + def __init__( + self, + name:str = "", title:str = "", + content: list[str] = [], + usages: list[dict[str]] = [], + experimental_flag: bool = False + ) -> None: + self.name = name + self.title = title + self.content = content + self.usages = [YosysCmdUsage(**u) for u in usages] + self.experimental_flag = experimental_flag + + @property + def source_file(self) -> str: + return "" + + @property + def source_line(self) -> int: + return 0 + +class YosysCmdGroupDocumenter(Documenter): + objtype = 'cmdgroup' + priority = 10 + object: tuple[str, list[str]] + lib_key = 'groups' + + option_spec = { + 'caption': autodoc.annotation_option, + 'members': autodoc.members_option, + 'source': autodoc.bool_option, + 'linenos': autodoc.bool_option, + } + + __cmd_lib: dict[str, list[str] | dict[str]] | None = None + @property + def cmd_lib(self) -> dict[str, list[str] | dict[str]]: + if not self.__cmd_lib: + self.__cmd_lib = {} + cmds_obj: dict[str, dict[str, list[str] | dict[str]]] + try: + with open(self.config.cmds_json, "r") as f: + cmds_obj = json.loads(f.read()) + except FileNotFoundError: + logger.warning( + f"unable to find cmd lib at {self.config.cmds_json}", + type = 'cmdref', + subtype = 'cmd_lib' + ) + else: + for (name, obj) in cmds_obj.get(self.lib_key, {}).items(): + self.__cmd_lib[name] = obj + return self.__cmd_lib + + @classmethod + def can_document_member( + cls, + member: Any, + membername: str, + isattr: bool, + parent: Any + ) -> bool: + return False + + def parse_name(self) -> bool: + if not self.options.caption: + self.content_indent = '' + self.fullname = self.modname = self.name + return True + + def import_object(self, raiseerror: bool = False) -> bool: + # get cmd + try: + self.object = (self.modname, self.cmd_lib[self.modname]) + except KeyError: + if raiseerror: + raise + return False + + self.real_modname = self.modname + return True + + def get_sourcename(self) -> str: + return self.env.doc2path(self.env.docname) + + def format_name(self) -> str: + return self.options.caption or '' + + def format_signature(self, **kwargs: Any) -> str: + return self.modname + + def add_directive_header(self, sig: str) -> None: + domain = getattr(self, 'domain', 'cmd') + directive = getattr(self, 'directivetype', 'group') + name = self.format_name() + sourcename = self.get_sourcename() + cmd_list = self.object + + # cmd definition + self.add_line(f'.. {domain}:{directive}:: {sig}', sourcename) + self.add_line(f' :caption: {name}', sourcename) + + if self.options.noindex: + self.add_line(' :noindex:', sourcename) + + def add_content(self, more_content: Any | None) -> None: + # groups have no native content + # add additional content (e.g. from document), if present + if more_content: + for line, src in zip(more_content.data, more_content.items): + self.add_line(line, src[0], src[1]) + + def filter_members( + self, + members: list[tuple[str, Any]], + want_all: bool + ) -> list[tuple[str, Any, bool]]: + return [(x[0], x[1], False) for x in members] + + def get_object_members( + self, + want_all: bool + ) -> tuple[bool, list[tuple[str, Any]]]: + ret: list[tuple[str, str]] = [] + + if want_all: + for member in self.object[1]: + ret.append((member, self.modname)) + else: + memberlist = self.options.members or [] + for name in memberlist: + if name in self.object: + ret.append((name, self.modname)) + else: + logger.warning(('unknown module mentioned in :members: option: ' + f'group {self.modname}, module {name}'), + type='cmdref') + + return False, ret + + def document_members(self, all_members: bool = False) -> None: + want_all = (all_members or + self.options.inherited_members or + self.options.members is autodoc.ALL) + # find out which members are documentable + members_check_module, members = self.get_object_members(want_all) + + # document non-skipped members + memberdocumenters: list[tuple[Documenter, bool]] = [] + for (mname, member, isattr) in self.filter_members(members, want_all): + classes = [cls for cls in self.documenters.values() + if cls.can_document_member(member, mname, isattr, self)] + if not classes: + # don't know how to document this member + continue + # prefer the documenter with the highest priority + classes.sort(key=lambda cls: cls.priority) + # give explicitly separated module name, so that members + # of inner classes can be documented + full_mname = self.format_signature() + '::' + mname + documenter = classes[-1](self.directive, full_mname, self.indent) + memberdocumenters.append((documenter, isattr)) + + member_order = self.options.member_order or self.config.autodoc_member_order + memberdocumenters = self.sort_members(memberdocumenters, member_order) + + for documenter, isattr in memberdocumenters: + documenter.generate( + all_members=True, real_modname=self.real_modname, + check_module=members_check_module and not isattr) + + def generate( + self, + more_content: Any | None = None, + real_modname: str | None = None, + check_module: bool = False, + all_members: bool = False + ) -> None: + if not self.parse_name(): + # need a cmd lib to import from + logger.warning( + f"don't know which cmd lib to import for autodocumenting {self.name}", + type = 'cmdref' + ) + return + + if not self.import_object(): + logger.warning( + f"unable to load {self.name} with {type(self)}", + type = 'cmdref' + ) + return + + # check __module__ of object (for members not given explicitly) + # if check_module: + # if not self.check_module(): + # return + + sourcename = self.get_sourcename() + self.add_line('', sourcename) + + # format the object's signature, if any + try: + sig = self.format_signature() + except Exception as exc: + logger.warning(('error while formatting signature for %s: %s'), + self.fullname, exc, type='cmdref') + return + + # generate the directive header and options, if applicable + self.add_directive_header(sig) + self.add_line('', sourcename) + + # e.g. the module directive doesn't have content + self.indent += self.content_indent + + # add all content (from docstrings, attribute docs etc.) + self.add_content(more_content) + + # document members, if possible + self.document_members(all_members) + +class YosysCmdDocumenter(YosysCmdGroupDocumenter): + objtype = 'cmd' + priority = 15 + object: YosysCmd + lib_key = 'cmds' + + @classmethod + def can_document_member( + cls, + member: Any, + membername: str, + isattr: bool, + parent: Any + ) -> bool: + if membername.startswith('$'): + return False + return isinstance(parent, YosysCmdGroupDocumenter) + + def parse_name(self) -> bool: + try: + matched = cmd_ext_sig_re.match(self.name) + modname, thing, attribute = matched.groups() + except AttributeError: + logger.warning(('invalid signature for auto%s (%r)') % (self.objtype, self.name), + type='cmdref') + return False + + self.modname = modname + self.attribute = attribute or '' + self.fullname = ((self.modname) + (thing or '')) + + return True + + def import_object(self, raiseerror: bool = False) -> bool: + if super().import_object(raiseerror): + self.object = YosysCmd(self.modname, **self.object[1]) + return True + return False + + def get_sourcename(self) -> str: + return self.object.source_file + + def format_name(self) -> str: + return self.object.name + + def format_signature(self, **kwargs: Any) -> str: + return self.fullname + self.attribute + + def add_directive_header(self, sig: str) -> None: + domain = getattr(self, 'domain', self.objtype) + directive = getattr(self, 'directivetype', 'def') + source_name = self.object.source_file + source_line = self.object.source_line + + # cmd definition + self.add_line(f'.. {domain}:{directive}:: {sig}', source_name, source_line) + + if self.options.noindex: + self.add_line(' :noindex:', source_name) + + def add_content(self, more_content: Any | None) -> None: + # set sourcename and add content from attribute documentation + domain = getattr(self, 'domain', self.objtype) + source_name = self.object.source_file + + for usage in self.object.usages: + self.add_line('', source_name) + if usage.signature: + self.add_line(f' .. {domain}:usage:: {self.name}::{usage.signature}', source_name) + self.add_line('', source_name) + for line in usage.description.splitlines(): + self.add_line(f' {line}', source_name) + self.add_line('', source_name) + if usage.options: + self.add_line(f' .. {domain}:optiongroup:: {self.name}::something', source_name) + self.add_line('', source_name) + for opt, desc in usage.options: + self.add_line(f' :option {opt}: {desc}', source_name) + self.add_line('', source_name) + for line in usage.postscript.splitlines(): + self.add_line(f' {line}', source_name) + self.add_line('', source_name) + + for line in self.object.content: + if line.startswith('..') and ':: ' in line: + line = line.replace(':: ', f':: {self.name}::', 1) + self.add_line(line, source_name) + + # add additional content (e.g. from document), if present + if more_content: + for line, src in zip(more_content.data, more_content.items): + self.add_line(line, src[0], src[1]) + + # fields + self.add_line('\n', source_name) + field_attrs = ["properties", ] + for field in field_attrs: + attr = getattr(self.object, field, []) + for val in attr: + self.add_line(f':{field} {val}:', source_name) + + def get_object_members( + self, + want_all: bool + ) -> tuple[bool, list[tuple[str, Any]]]: + + return False, [] + +class YosysCmdUsageDocumenter(YosysCmdDocumenter): + objtype = 'cmdusage' + priority = 20 + object: YosysCmdUsage + parent: YosysCmd + + def add_directive_header(self, sig: str) -> None: + domain = getattr(self, 'domain', 'cmd') + directive = getattr(self, 'directivetype', 'usage') + name = self.format_name() + sourcename = self.parent.source_file + cmd = self.parent + + # cmd definition + self.add_line(f'.. {domain}:{directive}:: {sig}', sourcename) + if self.object.signature: + self.add_line(f' :usage: {self.object.signature}', sourcename) + else: + self.add_line(f' :noindex:', sourcename) + # for usage in self.object.signature.splitlines(): + # self.add_line(f' :usage: {usage}', sourcename) + + # if self.options.linenos: + # self.add_line(f' :source: {cmd.source.split(":")[0]}', sourcename) + # else: + # self.add_line(f' :source: {cmd.source}', sourcename) + # self.add_line(f' :language: verilog', sourcename) + + if self.options.noindex: + self.add_line(' :noindex:', sourcename) + + def add_content(self, more_content: Any | None) -> None: + # set sourcename and add content from attribute documentation + sourcename = self.parent.source_file + startline = self.parent.source_line + + for line in self.object.description.splitlines(): + self.add_line(line, sourcename) + + # add additional content (e.g. from document), if present + if more_content: + for line, src in zip(more_content.data, more_content.items): + self.add_line(line, src[0], src[1]) + + def get_object_members( + self, + want_all: bool + ) -> tuple[bool, list[tuple[str, Any]]]: + return False, [] + +def setup(app: Sphinx) -> dict[str, Any]: + app.add_config_value('cmds_json', False, 'html', [Path, PosixPath, WindowsPath]) + app.setup_extension('sphinx.ext.autodoc') + app.add_autodocumenter(YosysCmdGroupDocumenter) + app.add_autodocumenter(YosysCmdDocumenter) + return { + 'version': '1', + 'parallel_read_safe': True, + } diff --git a/kernel/register.cc b/kernel/register.cc index a6d07cd8f..de70672a5 100644 --- a/kernel/register.cc +++ b/kernel/register.cc @@ -87,30 +87,6 @@ PRIVATE_NAMESPACE_END YOSYS_NAMESPACE_BEGIN -#define MAX_LINE_LEN 80 -void log_pass_str(const std::string &pass_str, int indent=0, bool leading_newline=false) { - if (pass_str.empty()) - return; - std::string indent_str(indent*4, ' '); - std::istringstream iss(pass_str); - if (leading_newline) - log("\n"); - for (std::string line; std::getline(iss, line);) { - log("%s", indent_str.c_str()); - auto curr_len = indent_str.length(); - std::istringstream lss(line); - for (std::string word; std::getline(lss, word, ' ');) { - if (curr_len + word.length() >= MAX_LINE_LEN) { - curr_len = 0; - log("\n%s", indent_str.c_str()); - } - log("%s ", word.c_str()); - curr_len += word.length() + 1; - } - log("\n"); - } -} - #define MAX_REG_COUNT 1000 bool echo_mode = false; @@ -123,7 +99,7 @@ std::map backend_register; std::vector Frontend::next_args; -Pass::Pass(std::string name, std::string short_help, const vector usages) : pass_name(name), short_help(short_help), pass_usages(usages) +Pass::Pass(std::string name, std::string short_help, const vector doc_string, const vector usages) : pass_name(name), short_help(short_help), doc_string(doc_string), pass_usages(usages) { next_queued_pass = first_queued_pass; first_queued_pass = this; @@ -196,6 +172,37 @@ void Pass::post_execute(Pass::pre_post_exec_state_t state) current_pass->runtime_ns -= time_ns; } +#define MAX_LINE_LEN 80 +void log_pass_str(const std::string &pass_str, std::string indent_str, bool leading_newline=false) { + if (pass_str.empty()) + return; + std::istringstream iss(pass_str); + if (leading_newline) + log("\n"); + for (std::string line; std::getline(iss, line);) { + log("%s", indent_str.c_str()); + auto curr_len = indent_str.length(); + std::istringstream lss(line); + for (std::string word; std::getline(lss, word, ' ');) { + while (word[0] == '`' && word.back() == '`') + word = word.substr(1, word.length()-2); + if (curr_len + word.length() >= MAX_LINE_LEN-1) { + curr_len = 0; + log("\n%s", indent_str.c_str()); + } + if (word.length()) { + log("%s ", word.c_str()); + curr_len += word.length() + 1; + } + } + log("\n"); + } +} +void log_pass_str(const std::string &pass_str, int indent=0, bool leading_newline=false) { + std::string indent_str(indent*4, ' '); + log_pass_str(pass_str, indent_str, leading_newline); +} + void Pass::help() { if (HasUsages()) { @@ -209,6 +216,28 @@ void Pass::help() log_pass_str(usage.postscript, 0, true); } log("\n"); + } else if (HasDocstring()) { + log("\n"); + auto print_empty = true; + for (auto doc_line : doc_string) { + if (doc_line.find("..") == 0 && doc_line.find(":: ") != std::string::npos) { + auto command_pos = doc_line.find(":: "); + auto command_str = doc_line.substr(0, command_pos); + if (command_str.compare(".. cmd:usage") == 0) { + log_pass_str(doc_line.substr(command_pos+3), 1); + } else { + print_empty = false; + } + } else if (doc_line.length()) { + std::size_t first_pos = doc_line.find_first_not_of(" \t"); + auto indent_str = doc_line.substr(0, first_pos); + log_pass_str(doc_line, indent_str); + print_empty = true; + } else if (print_empty) { + log("\n"); + } + } + log("\n"); } else { log("\n"); log("No help message for command `%s'.\n", pass_name.c_str()); @@ -948,7 +977,7 @@ struct HelpPass : public Pass { vector usages; auto experimental_flag = pass->experimental_flag; - if (pass->HasUsages()) { + if (pass->HasUsages() || pass->HasDocstring()) { for (auto usage : pass->pass_usages) usages.push_back(usage); } else { @@ -1045,23 +1074,28 @@ struct HelpPass : public Pass { // write to json json.name(name.c_str()); json.begin_object(); json.entry("title", title); - json.name("usages"); json.begin_array(); - for (auto usage : usages) { - json.begin_object(); - json.entry("signature", usage.signature); - json.entry("description", usage.description); - json.name("options"); json.begin_array(); - for (auto option : usage.options) { - json.begin_array(); - json.value(option.keyword); - json.value(option.description); + if (pass->HasDocstring()) { + json.entry("content", pass->doc_string); + } + if (usages.size()) { + json.name("usages"); json.begin_array(); + for (auto usage : usages) { + json.begin_object(); + json.entry("signature", usage.signature); + json.entry("description", usage.description); + json.name("options"); json.begin_array(); + for (auto option : usage.options) { + json.begin_array(); + json.value(option.keyword); + json.value(option.description); + json.end_array(); + } json.end_array(); + json.entry("postscript", usage.postscript); + json.end_object(); } json.end_array(); - json.entry("postscript", usage.postscript); - json.end_object(); } - json.end_array(); json.entry("experimental_flag", experimental_flag); json.end_object(); } diff --git a/kernel/register.h b/kernel/register.h index 31167d532..64b2ce42f 100644 --- a/kernel/register.h +++ b/kernel/register.h @@ -40,8 +40,10 @@ struct PassUsageBlock { struct Pass { std::string pass_name, short_help; + const vector doc_string; const vector pass_usages; Pass(std::string name, std::string short_help = "** document me **", + const vector doc_string = {}, const vector usages = {}); virtual ~Pass(); @@ -61,6 +63,10 @@ struct Pass return !pass_usages.empty(); } + bool HasDocstring() { + return !doc_string.empty(); + } + struct pre_post_exec_state_t { Pass *parent_pass; int64_t begin_ns;