Docs: Proto doc_string approach for cmd help

Add `doc_string` field to `Pass` constructor
Add `docs/util/newcmdref.py` to contain command domain
Update `docs/util/cmdref.py` with `cmd:usage` and `cmd:optiongroup` for describing commands.
Functional, but WIP.
This commit is contained in:
Krystine Sherwin 2025-01-13 15:19:10 +13:00
parent 023a1e3c03
commit 2b35c4dccf
No known key found for this signature in database
5 changed files with 635 additions and 43 deletions

View File

@ -84,6 +84,8 @@ extensions.append('util.cmdref')
extensions.append('sphinx.ext.autodoc')
extensions.append('util.cellref')
cells_json = Path(__file__).parent / 'generated' / 'cells.json'
extensions.append('util.newcmdref')
cmds_json = Path(__file__).parent / 'generated' / 'cmds.json'
from sphinx.application import Sphinx
def setup(app: Sphinx) -> None:

View File

@ -4,10 +4,12 @@ from __future__ import annotations
import re
from typing import cast
import warnings
from docutils import nodes
from docutils.nodes import Node, Element, system_message
from docutils.nodes import Node, Element
from docutils.parsers.rst import directives
from docutils.parsers.rst.roles import GenericRole
from docutils.parsers.rst.states import Inliner
from sphinx.application import Sphinx
from sphinx.domains import Domain, Index
@ -17,7 +19,7 @@ from sphinx.roles import XRefRole
from sphinx.directives import ObjectDescription
from sphinx.directives.code import container_wrapper
from sphinx.util.nodes import make_refnode
from sphinx.util.docfields import Field
from sphinx.util.docfields import Field, GroupedField
from sphinx import addnodes
class TocNode(ObjectDescription):
@ -63,10 +65,15 @@ class CommandNode(TocNode):
name = 'cmd'
required_arguments = 1
option_spec = {
option_spec = TocNode.option_spec.copy()
option_spec.update({
'title': directives.unchanged,
'tags': directives.unchanged
}
})
doc_field_types = [
GroupedField('opts', label='Options', names=('option', 'options', 'opt', 'opts')),
]
def handle_signature(self, sig, signode: addnodes.desc_signature):
signode['fullname'] = sig
@ -93,6 +100,120 @@ class CommandNode(TocNode):
idx,
0))
class CommandUsageNode(TocNode):
"""A custom node that describes command usages"""
name = 'cmdusage'
option_spec = TocNode.option_spec
option_spec.update({
'usage': directives.unchanged,
})
doc_field_types = [
GroupedField('opts', label='Options', names=('option', 'options', 'opt', 'opts')),
]
def handle_signature(self, sig: str, signode: addnodes.desc_signature):
try:
cmd, use = sig.split('::')
except ValueError:
cmd, use = sig, ''
signode['fullname'] = sig
usage = self.options.get('usage', use or sig)
if usage:
signode['tocname'] = usage
signode += addnodes.desc_name(text=usage)
return signode['fullname']
def add_target_and_index(
self,
name: str,
sig: str,
signode: addnodes.desc_signature
) -> None:
idx = ".".join(name.split("::"))
signode['ids'].append(idx)
if 'noindex' not in self.options:
tocname: str = signode.get('tocname', name)
objs = self.env.domaindata[self.domain]['objects']
# (name, sig, typ, docname, anchor, prio)
objs.append((name,
tocname,
type(self).name,
self.env.docname,
idx,
1))
class CommandOptionGroupNode(TocNode):
"""A custom node that describes a group of related options"""
name = 'cmdoptiongroup'
option_spec = TocNode.option_spec
doc_field_types = [
Field('opt', ('option',), label='', rolename='option')
]
def handle_signature(self, sig: str, signode: addnodes.desc_signature):
try:
cmd, name = sig.split('::')
except ValueError:
cmd, name = '', sig
signode['fullname'] = sig
signode['tocname'] = name
signode += addnodes.desc_name(text=name)
return signode['fullname']
def add_target_and_index(
self,
name: str,
sig: str,
signode: addnodes.desc_signature
) -> None:
idx = ".".join(name.split("::"))
signode['ids'].append(idx)
if 'noindex' not in self.options:
tocname: str = signode.get('tocname', name)
objs = self.env.domaindata[self.domain]['objects']
# (name, sig, typ, docname, anchor, prio)
objs.append((name,
tocname,
type(self).name,
self.env.docname,
idx,
1))
def transform_content(self, contentnode: addnodes.desc_content) -> None:
"""hack `:option -thing: desc` into a proper option list"""
newchildren = []
for node in contentnode:
newnode = node
if isinstance(node, nodes.field_list):
newnode = nodes.option_list()
for field in node:
is_option = False
option_list_item = nodes.option_list_item()
for child in field:
if isinstance(child, nodes.field_name):
option_group = nodes.option_group()
option_list_item += option_group
option = nodes.option()
option_group += option
name, text = child.rawsource.split(' ', 1)
is_option = name == 'option'
option += nodes.option_string(text=text)
if not is_option: warnings.warn(f'unexpected option \'{name}\' in {field.source}')
elif isinstance(child, nodes.field_body):
description = nodes.description()
description += child.children
option_list_item += description
if is_option:
newnode += option_list_item
newchildren.append(newnode)
contentnode.children = newchildren
class PropNode(TocNode):
name = 'prop'
fieldname = 'props'
@ -517,6 +638,8 @@ class CommandDomain(Domain):
directives = {
'def': CommandNode,
'usage': CommandUsageNode,
'optiongroup': CommandOptionGroupNode,
}
indices = {

427
docs/util/newcmdref.py Normal file
View File

@ -0,0 +1,427 @@
#!/usr/bin/env python3
from __future__ import annotations
from dataclasses import dataclass
import json
from pathlib import Path, PosixPath, WindowsPath
import re
from typing import Any
from sphinx.application import Sphinx
from sphinx.ext import autodoc
from sphinx.ext.autodoc import Documenter
from sphinx.util import logging
logger = logging.getLogger(__name__)
# cmd signature
cmd_ext_sig_re = re.compile(
r'''^ ([\w$._]+?) # module name
(?:\.([\w_]+))? # optional: thing name
(::[\w_]+)? # attribute
\s* $ # and nothing more
''', re.VERBOSE)
@dataclass
class YosysCmdUsage:
signature: str
description: str
options: list[tuple[str,str]]
postscript: str
class YosysCmd:
name: str
title: str
content: list[str]
usages: list[YosysCmdUsage]
experimental_flag: bool
def __init__(
self,
name:str = "", title:str = "",
content: list[str] = [],
usages: list[dict[str]] = [],
experimental_flag: bool = False
) -> None:
self.name = name
self.title = title
self.content = content
self.usages = [YosysCmdUsage(**u) for u in usages]
self.experimental_flag = experimental_flag
@property
def source_file(self) -> str:
return ""
@property
def source_line(self) -> int:
return 0
class YosysCmdGroupDocumenter(Documenter):
objtype = 'cmdgroup'
priority = 10
object: tuple[str, list[str]]
lib_key = 'groups'
option_spec = {
'caption': autodoc.annotation_option,
'members': autodoc.members_option,
'source': autodoc.bool_option,
'linenos': autodoc.bool_option,
}
__cmd_lib: dict[str, list[str] | dict[str]] | None = None
@property
def cmd_lib(self) -> dict[str, list[str] | dict[str]]:
if not self.__cmd_lib:
self.__cmd_lib = {}
cmds_obj: dict[str, dict[str, list[str] | dict[str]]]
try:
with open(self.config.cmds_json, "r") as f:
cmds_obj = json.loads(f.read())
except FileNotFoundError:
logger.warning(
f"unable to find cmd lib at {self.config.cmds_json}",
type = 'cmdref',
subtype = 'cmd_lib'
)
else:
for (name, obj) in cmds_obj.get(self.lib_key, {}).items():
self.__cmd_lib[name] = obj
return self.__cmd_lib
@classmethod
def can_document_member(
cls,
member: Any,
membername: str,
isattr: bool,
parent: Any
) -> bool:
return False
def parse_name(self) -> bool:
if not self.options.caption:
self.content_indent = ''
self.fullname = self.modname = self.name
return True
def import_object(self, raiseerror: bool = False) -> bool:
# get cmd
try:
self.object = (self.modname, self.cmd_lib[self.modname])
except KeyError:
if raiseerror:
raise
return False
self.real_modname = self.modname
return True
def get_sourcename(self) -> str:
return self.env.doc2path(self.env.docname)
def format_name(self) -> str:
return self.options.caption or ''
def format_signature(self, **kwargs: Any) -> str:
return self.modname
def add_directive_header(self, sig: str) -> None:
domain = getattr(self, 'domain', 'cmd')
directive = getattr(self, 'directivetype', 'group')
name = self.format_name()
sourcename = self.get_sourcename()
cmd_list = self.object
# cmd definition
self.add_line(f'.. {domain}:{directive}:: {sig}', sourcename)
self.add_line(f' :caption: {name}', sourcename)
if self.options.noindex:
self.add_line(' :noindex:', sourcename)
def add_content(self, more_content: Any | None) -> None:
# groups have no native content
# add additional content (e.g. from document), if present
if more_content:
for line, src in zip(more_content.data, more_content.items):
self.add_line(line, src[0], src[1])
def filter_members(
self,
members: list[tuple[str, Any]],
want_all: bool
) -> list[tuple[str, Any, bool]]:
return [(x[0], x[1], False) for x in members]
def get_object_members(
self,
want_all: bool
) -> tuple[bool, list[tuple[str, Any]]]:
ret: list[tuple[str, str]] = []
if want_all:
for member in self.object[1]:
ret.append((member, self.modname))
else:
memberlist = self.options.members or []
for name in memberlist:
if name in self.object:
ret.append((name, self.modname))
else:
logger.warning(('unknown module mentioned in :members: option: '
f'group {self.modname}, module {name}'),
type='cmdref')
return False, ret
def document_members(self, all_members: bool = False) -> None:
want_all = (all_members or
self.options.inherited_members or
self.options.members is autodoc.ALL)
# find out which members are documentable
members_check_module, members = self.get_object_members(want_all)
# document non-skipped members
memberdocumenters: list[tuple[Documenter, bool]] = []
for (mname, member, isattr) in self.filter_members(members, want_all):
classes = [cls for cls in self.documenters.values()
if cls.can_document_member(member, mname, isattr, self)]
if not classes:
# don't know how to document this member
continue
# prefer the documenter with the highest priority
classes.sort(key=lambda cls: cls.priority)
# give explicitly separated module name, so that members
# of inner classes can be documented
full_mname = self.format_signature() + '::' + mname
documenter = classes[-1](self.directive, full_mname, self.indent)
memberdocumenters.append((documenter, isattr))
member_order = self.options.member_order or self.config.autodoc_member_order
memberdocumenters = self.sort_members(memberdocumenters, member_order)
for documenter, isattr in memberdocumenters:
documenter.generate(
all_members=True, real_modname=self.real_modname,
check_module=members_check_module and not isattr)
def generate(
self,
more_content: Any | None = None,
real_modname: str | None = None,
check_module: bool = False,
all_members: bool = False
) -> None:
if not self.parse_name():
# need a cmd lib to import from
logger.warning(
f"don't know which cmd lib to import for autodocumenting {self.name}",
type = 'cmdref'
)
return
if not self.import_object():
logger.warning(
f"unable to load {self.name} with {type(self)}",
type = 'cmdref'
)
return
# check __module__ of object (for members not given explicitly)
# if check_module:
# if not self.check_module():
# return
sourcename = self.get_sourcename()
self.add_line('', sourcename)
# format the object's signature, if any
try:
sig = self.format_signature()
except Exception as exc:
logger.warning(('error while formatting signature for %s: %s'),
self.fullname, exc, type='cmdref')
return
# generate the directive header and options, if applicable
self.add_directive_header(sig)
self.add_line('', sourcename)
# e.g. the module directive doesn't have content
self.indent += self.content_indent
# add all content (from docstrings, attribute docs etc.)
self.add_content(more_content)
# document members, if possible
self.document_members(all_members)
class YosysCmdDocumenter(YosysCmdGroupDocumenter):
objtype = 'cmd'
priority = 15
object: YosysCmd
lib_key = 'cmds'
@classmethod
def can_document_member(
cls,
member: Any,
membername: str,
isattr: bool,
parent: Any
) -> bool:
if membername.startswith('$'):
return False
return isinstance(parent, YosysCmdGroupDocumenter)
def parse_name(self) -> bool:
try:
matched = cmd_ext_sig_re.match(self.name)
modname, thing, attribute = matched.groups()
except AttributeError:
logger.warning(('invalid signature for auto%s (%r)') % (self.objtype, self.name),
type='cmdref')
return False
self.modname = modname
self.attribute = attribute or ''
self.fullname = ((self.modname) + (thing or ''))
return True
def import_object(self, raiseerror: bool = False) -> bool:
if super().import_object(raiseerror):
self.object = YosysCmd(self.modname, **self.object[1])
return True
return False
def get_sourcename(self) -> str:
return self.object.source_file
def format_name(self) -> str:
return self.object.name
def format_signature(self, **kwargs: Any) -> str:
return self.fullname + self.attribute
def add_directive_header(self, sig: str) -> None:
domain = getattr(self, 'domain', self.objtype)
directive = getattr(self, 'directivetype', 'def')
source_name = self.object.source_file
source_line = self.object.source_line
# cmd definition
self.add_line(f'.. {domain}:{directive}:: {sig}', source_name, source_line)
if self.options.noindex:
self.add_line(' :noindex:', source_name)
def add_content(self, more_content: Any | None) -> None:
# set sourcename and add content from attribute documentation
domain = getattr(self, 'domain', self.objtype)
source_name = self.object.source_file
for usage in self.object.usages:
self.add_line('', source_name)
if usage.signature:
self.add_line(f' .. {domain}:usage:: {self.name}::{usage.signature}', source_name)
self.add_line('', source_name)
for line in usage.description.splitlines():
self.add_line(f' {line}', source_name)
self.add_line('', source_name)
if usage.options:
self.add_line(f' .. {domain}:optiongroup:: {self.name}::something', source_name)
self.add_line('', source_name)
for opt, desc in usage.options:
self.add_line(f' :option {opt}: {desc}', source_name)
self.add_line('', source_name)
for line in usage.postscript.splitlines():
self.add_line(f' {line}', source_name)
self.add_line('', source_name)
for line in self.object.content:
if line.startswith('..') and ':: ' in line:
line = line.replace(':: ', f':: {self.name}::', 1)
self.add_line(line, source_name)
# add additional content (e.g. from document), if present
if more_content:
for line, src in zip(more_content.data, more_content.items):
self.add_line(line, src[0], src[1])
# fields
self.add_line('\n', source_name)
field_attrs = ["properties", ]
for field in field_attrs:
attr = getattr(self.object, field, [])
for val in attr:
self.add_line(f':{field} {val}:', source_name)
def get_object_members(
self,
want_all: bool
) -> tuple[bool, list[tuple[str, Any]]]:
return False, []
class YosysCmdUsageDocumenter(YosysCmdDocumenter):
objtype = 'cmdusage'
priority = 20
object: YosysCmdUsage
parent: YosysCmd
def add_directive_header(self, sig: str) -> None:
domain = getattr(self, 'domain', 'cmd')
directive = getattr(self, 'directivetype', 'usage')
name = self.format_name()
sourcename = self.parent.source_file
cmd = self.parent
# cmd definition
self.add_line(f'.. {domain}:{directive}:: {sig}', sourcename)
if self.object.signature:
self.add_line(f' :usage: {self.object.signature}', sourcename)
else:
self.add_line(f' :noindex:', sourcename)
# for usage in self.object.signature.splitlines():
# self.add_line(f' :usage: {usage}', sourcename)
# if self.options.linenos:
# self.add_line(f' :source: {cmd.source.split(":")[0]}', sourcename)
# else:
# self.add_line(f' :source: {cmd.source}', sourcename)
# self.add_line(f' :language: verilog', sourcename)
if self.options.noindex:
self.add_line(' :noindex:', sourcename)
def add_content(self, more_content: Any | None) -> None:
# set sourcename and add content from attribute documentation
sourcename = self.parent.source_file
startline = self.parent.source_line
for line in self.object.description.splitlines():
self.add_line(line, sourcename)
# add additional content (e.g. from document), if present
if more_content:
for line, src in zip(more_content.data, more_content.items):
self.add_line(line, src[0], src[1])
def get_object_members(
self,
want_all: bool
) -> tuple[bool, list[tuple[str, Any]]]:
return False, []
def setup(app: Sphinx) -> dict[str, Any]:
app.add_config_value('cmds_json', False, 'html', [Path, PosixPath, WindowsPath])
app.setup_extension('sphinx.ext.autodoc')
app.add_autodocumenter(YosysCmdGroupDocumenter)
app.add_autodocumenter(YosysCmdDocumenter)
return {
'version': '1',
'parallel_read_safe': True,
}

View File

@ -87,30 +87,6 @@ PRIVATE_NAMESPACE_END
YOSYS_NAMESPACE_BEGIN
#define MAX_LINE_LEN 80
void log_pass_str(const std::string &pass_str, int indent=0, bool leading_newline=false) {
if (pass_str.empty())
return;
std::string indent_str(indent*4, ' ');
std::istringstream iss(pass_str);
if (leading_newline)
log("\n");
for (std::string line; std::getline(iss, line);) {
log("%s", indent_str.c_str());
auto curr_len = indent_str.length();
std::istringstream lss(line);
for (std::string word; std::getline(lss, word, ' ');) {
if (curr_len + word.length() >= MAX_LINE_LEN) {
curr_len = 0;
log("\n%s", indent_str.c_str());
}
log("%s ", word.c_str());
curr_len += word.length() + 1;
}
log("\n");
}
}
#define MAX_REG_COUNT 1000
bool echo_mode = false;
@ -123,7 +99,7 @@ std::map<std::string, Backend*> backend_register;
std::vector<std::string> Frontend::next_args;
Pass::Pass(std::string name, std::string short_help, const vector<PassUsageBlock> usages) : pass_name(name), short_help(short_help), pass_usages(usages)
Pass::Pass(std::string name, std::string short_help, const vector<std::string> doc_string, const vector<PassUsageBlock> usages) : pass_name(name), short_help(short_help), doc_string(doc_string), pass_usages(usages)
{
next_queued_pass = first_queued_pass;
first_queued_pass = this;
@ -196,6 +172,37 @@ void Pass::post_execute(Pass::pre_post_exec_state_t state)
current_pass->runtime_ns -= time_ns;
}
#define MAX_LINE_LEN 80
void log_pass_str(const std::string &pass_str, std::string indent_str, bool leading_newline=false) {
if (pass_str.empty())
return;
std::istringstream iss(pass_str);
if (leading_newline)
log("\n");
for (std::string line; std::getline(iss, line);) {
log("%s", indent_str.c_str());
auto curr_len = indent_str.length();
std::istringstream lss(line);
for (std::string word; std::getline(lss, word, ' ');) {
while (word[0] == '`' && word.back() == '`')
word = word.substr(1, word.length()-2);
if (curr_len + word.length() >= MAX_LINE_LEN-1) {
curr_len = 0;
log("\n%s", indent_str.c_str());
}
if (word.length()) {
log("%s ", word.c_str());
curr_len += word.length() + 1;
}
}
log("\n");
}
}
void log_pass_str(const std::string &pass_str, int indent=0, bool leading_newline=false) {
std::string indent_str(indent*4, ' ');
log_pass_str(pass_str, indent_str, leading_newline);
}
void Pass::help()
{
if (HasUsages()) {
@ -209,6 +216,28 @@ void Pass::help()
log_pass_str(usage.postscript, 0, true);
}
log("\n");
} else if (HasDocstring()) {
log("\n");
auto print_empty = true;
for (auto doc_line : doc_string) {
if (doc_line.find("..") == 0 && doc_line.find(":: ") != std::string::npos) {
auto command_pos = doc_line.find(":: ");
auto command_str = doc_line.substr(0, command_pos);
if (command_str.compare(".. cmd:usage") == 0) {
log_pass_str(doc_line.substr(command_pos+3), 1);
} else {
print_empty = false;
}
} else if (doc_line.length()) {
std::size_t first_pos = doc_line.find_first_not_of(" \t");
auto indent_str = doc_line.substr(0, first_pos);
log_pass_str(doc_line, indent_str);
print_empty = true;
} else if (print_empty) {
log("\n");
}
}
log("\n");
} else {
log("\n");
log("No help message for command `%s'.\n", pass_name.c_str());
@ -948,7 +977,7 @@ struct HelpPass : public Pass {
vector<PassUsageBlock> usages;
auto experimental_flag = pass->experimental_flag;
if (pass->HasUsages()) {
if (pass->HasUsages() || pass->HasDocstring()) {
for (auto usage : pass->pass_usages)
usages.push_back(usage);
} else {
@ -1045,23 +1074,28 @@ struct HelpPass : public Pass {
// write to json
json.name(name.c_str()); json.begin_object();
json.entry("title", title);
json.name("usages"); json.begin_array();
for (auto usage : usages) {
json.begin_object();
json.entry("signature", usage.signature);
json.entry("description", usage.description);
json.name("options"); json.begin_array();
for (auto option : usage.options) {
json.begin_array();
json.value(option.keyword);
json.value(option.description);
if (pass->HasDocstring()) {
json.entry("content", pass->doc_string);
}
if (usages.size()) {
json.name("usages"); json.begin_array();
for (auto usage : usages) {
json.begin_object();
json.entry("signature", usage.signature);
json.entry("description", usage.description);
json.name("options"); json.begin_array();
for (auto option : usage.options) {
json.begin_array();
json.value(option.keyword);
json.value(option.description);
json.end_array();
}
json.end_array();
json.entry("postscript", usage.postscript);
json.end_object();
}
json.end_array();
json.entry("postscript", usage.postscript);
json.end_object();
}
json.end_array();
json.entry("experimental_flag", experimental_flag);
json.end_object();
}

View File

@ -40,8 +40,10 @@ struct PassUsageBlock {
struct Pass
{
std::string pass_name, short_help;
const vector<std::string> doc_string;
const vector<PassUsageBlock> pass_usages;
Pass(std::string name, std::string short_help = "** document me **",
const vector<std::string> doc_string = {},
const vector<PassUsageBlock> usages = {});
virtual ~Pass();
@ -61,6 +63,10 @@ struct Pass
return !pass_usages.empty();
}
bool HasDocstring() {
return !doc_string.empty();
}
struct pre_post_exec_state_t {
Pass *parent_pass;
int64_t begin_ns;