Commit 9f73153f authored by zhanggzh's avatar zhanggzh
Browse files

add dtk24.04 code

parent eb77376e
"""Hack autodoc to get more fine-grained docstring rendering contol.
autodoc and autosummary didn't expose many of their controls to sphinx users via config.
To customize them, the "correct" approach seems to copy and paste all their code and rewrite some part.
To avoid doing this, I monkey-patched some of the functions to keep the changes minimal.
Note that some of them are related to sphinx internal APIs, which can be broken when sphinx got upgraded.
Try to keep them updated, or pin to a particular sphinx version.
"""
import inspect
import os
from typing import List, Tuple, List
import sphinx
from docutils import nodes
from docutils.nodes import Node
class ClassNewBlacklistPatch:
"""Force some classes to skip ``__new__`` when generating signature."""
original = None
def restore(self, *args, **kwargs):
assert self.original is not None
sphinx.ext.autodoc._CLASS_NEW_BLACKLIST = self.original
def patch(self, *args, **kwargs):
self.original = sphinx.ext.autodoc._CLASS_NEW_BLACKLIST
blacklist = []
import nni.retiarii.nn.pytorch
for name in dir(nni.retiarii.nn.pytorch):
obj = getattr(nni.retiarii.nn.pytorch, name)
if inspect.isclass(obj):
new_name = "{0.__module__}.{0.__qualname__}".format(obj.__new__)
if new_name not in blacklist:
blacklist.append(new_name)
sphinx.ext.autodoc._CLASS_NEW_BLACKLIST = self.original + blacklist
def disable_trace_patch(*args, **kwargs):
"""Disable trace by setting an environment variable."""
os.environ['NNI_TRACE_FLAG'] = 'DISABLE'
def trial_tool_import_patch(*args, **kwargs):
"""Insert dummy trial tool variable to ensure trial_tool can be imported.
See nni/tools/trial_tool/constants.py
"""
os.environ.update({
'NNI_OUTPUT_DIR': '/tmp',
'NNI_PLATFORM': 'unittest',
'NNI_SYS_DIR': '/tmp',
'NNI_TRIAL_JOB_ID': 'dummy',
'NNI_EXP_ID': 'dummy',
'MULTI_PHASE': 'dummy'
})
class AutoSummaryPatch:
"""Ignore certain files as they are completely un-importable. It patches:
- find_autosummary_in_files: Some modules cannot be imported at all due to dependency issues or some special design.
They need to skipped when running autosummary generate.
- Autosummary.get_table: The original autosummary creates an index for each module, and the module links in autosummary table
points to the corresponding generated module page (by using ``:py:module:xxx``). This doesn't work for us,
because we have used automodule else (other than autosummary) in our docs, and to avoid duplicate index,
we have to set ``:noindex:`` in autosummary template (see docs/templates/autosummary/module.rst).
This breaks most of the links, where they fail to link to generated module page by using index.
We here update the python domain role, to a general domain role (``:doc:``), and link to the page directly.
"""
find_autosummary_original = None
get_table_original = None
def restore(self, *args, **kwargs):
assert self.find_autosummary_original is not None and self.get_table_original is not None
sphinx.ext.autosummary.generate.find_autosummary_in_files = self.find_autosummary_original
sphinx.ext.autosummary.Autosummary.get_table = self.get_table_original
def patch(self, app, config):
from sphinx.ext.autosummary import Autosummary
from sphinx.ext.autosummary.generate import AutosummaryEntry
self.find_autosummary_original = sphinx.ext.autosummary.generate.find_autosummary_in_files
self.get_table_original = Autosummary.get_table
def find_autosummary_in_files(filenames: List[str]) -> List[AutosummaryEntry]:
items: List[AutosummaryEntry] = self.find_autosummary_original(filenames)
items = [item for item in items if item.name not in config.autosummary_mock_imports]
return items
def get_table(autosummary, items: List[Tuple[str, str, str, str]]) -> List[Node]:
col_spec, autosummary_table = self.get_table_original(autosummary, items)
if 'toctree' in autosummary.options:
# probably within modules
table = autosummary_table[0]
tgroup = table[0]
tbody = tgroup[-1]
for row in tbody:
entry = row[0]
paragraph = entry[0]
pending_xref = paragraph[0]
# get the reference path and check whether it has been generated
# if path to reference is changed, this should also be changed
reftarget_path = 'reference/_modules/' + pending_xref['reftarget']
if reftarget_path in autosummary.env.found_docs:
# make :py:obj:`xxx` looks like a :doc:`xxx`
pending_xref['refdomain'] = 'std'
pending_xref['reftype'] = 'doc'
pending_xref['refexplicit'] = False
pending_xref['refwarn'] = True
pending_xref['reftarget'] = '/' + reftarget_path
# a special tag to enable `ResolveDocPatch`
pending_xref['refkeepformat'] = True
return [col_spec, autosummary_table]
sphinx.ext.autosummary.generate.find_autosummary_in_files = find_autosummary_in_files
sphinx.ext.autosummary.Autosummary.get_table = get_table
class ResolveDocPatch:
"""Original :doc: role throws away all the format, and keep raw text only.
We wish to keep module names literal. This patch is to keep literal format in :doc: resolver."""
original = None
def restore(self, *args, **kwargs):
assert self.original is not None
sphinx.domains.std.StandardDomain._resolve_doc_xref = self.original
def patch(self, *args, **kwargs):
self.original = sphinx.domains.std.StandardDomain._resolve_doc_xref
def doc_xref_resolver(std_domain, env, fromdocname, builder, typ, target, node, contnode):
if not node.get('refkeepformat'):
# redirect to original implementation to make it safer
return self.original(std_domain, env, fromdocname, builder, typ, target, node, contnode)
# directly reference to document by source name; can be absolute or relative
from sphinx.domains.std import docname_join, make_refnode
refdoc = node.get('refdoc', fromdocname)
docname = docname_join(refdoc, node['reftarget'])
if docname not in env.all_docs:
return None
else:
innernode = node[0] # no astext here, to keep literal intact
return make_refnode(builder, fromdocname, docname, None, innernode)
sphinx.domains.std.StandardDomain._resolve_doc_xref = doc_xref_resolver
def setup(app):
# See life-cycle of sphinx app here:
# https://www.sphinx-doc.org/en/master/extdev/appapi.html#sphinx-core-events
patch = ClassNewBlacklistPatch()
app.connect('env-before-read-docs', patch.patch)
app.connect('env-merge-info', patch.restore)
patch = ResolveDocPatch()
app.connect('env-before-read-docs', patch.patch)
app.connect('env-merge-info', patch.restore)
app.connect('env-before-read-docs', disable_trace_patch)
# autosummary generate happens at builder-inited
app.connect('config-inited', trial_tool_import_patch)
autosummary_patch = AutoSummaryPatch()
app.connect('config-inited', autosummary_patch.patch)
app.connect('env-merge-info', autosummary_patch.restore)
"""
Make sure pages that contain toctree only has a toctree,
because, if our theme is used, other contents will not be visible.
"""
import re
from docutils import nodes
from sphinx.application import Sphinx
from sphinx.addnodes import toctree
from sphinx.util.logging import getLogger
logger = getLogger('toctree_check')
def _strip_compound(node):
if isinstance(node, nodes.compound):
return _strip_compound(node[0])
return node
def toctree_check(app: Sphinx, doctree: nodes.document, docname: str):
whitelist = app.config.toctree_check_whitelist
if docname in whitelist:
return
# Scan top-level nodes
has_toctree = False
other_types = []
for i in range(len(doctree[0])):
node = doctree[0][i]
if isinstance(_strip_compound(node), toctree):
has_toctree = True
elif isinstance(_strip_compound(node), nodes.title):
# Allow title
pass
else:
other_types.append(type(_strip_compound(node)))
if has_toctree and other_types:
# We don't allow a document with toctree to have other types of contents
logger.warning('Expect a toctree document to contain only a toctree, '
'but found other types of contents: %s', str(set(other_types)),
location=docname)
def setup(app):
app.connect('doctree-resolved', toctree_check)
app.add_config_value('toctree_check_whitelist', [], True)
"""Creating hard links for tutorials in each individual topics."""
import os
import re
HEADER = """.. THIS FILE IS A COPY OF {} WITH MODIFICATIONS.
.. TO MAKE ONE TUTORIAL APPEAR IN MULTIPLE PLACES.
"""
def flatten_filename(filename):
return filename.replace('/', '_').replace('.', '_')
def copy_tutorials(app):
# TODO: use sphinx logger
print('[tutorial links] copy tutorials...')
for src, tar in app.config.tutorials_copy_list:
target_path = os.path.join(app.srcdir, tar)
content = open(os.path.join(app.srcdir, src)).read()
# Add a header
content = HEADER.format(src) + content
# Add a prefix to labels to avoid duplicates.
label_map = {}
# find all anchors: https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html
# but not hyperlinks: https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html#external-links
for prefix, label_name in list(re.findall(r'(\.\.\s*_)(.*?)\:\s*\n', content)):
label_map[label_name] = flatten_filename(tar) + '_' + label_name
# anchor
content = content.replace(prefix + label_name + ':', prefix + label_map[label_name] + ':')
# :ref:`xxx`
content = content.replace(f':ref:`{label_name}`', f':ref:`{label_map[label_name]}')
# :ref:`yyy <xxx>`
content = re.sub(r"(\:ref\:`.*?\<)" + label_name + r"(\>`)", r'\1' + label_map[label_name] + r'\2', content)
open(target_path, 'w').write(content)
def setup(app):
# See life-cycle of sphinx app here:
# https://www.sphinx-doc.org/en/master/extdev/appapi.html#sphinx-core-events
app.connect('builder-inited', copy_tutorials)
app.add_config_value('tutorials_copy_list', [], True, [list])
<svg id="SvgjsSvg1006" width="709" height="537.0000305175781" xmlns="http://www.w3.org/2000/svg" version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:svgjs="http://svgjs.com/svgjs"><defs id="SvgjsDefs1007"><marker id="SvgjsMarker1034" markerWidth="16" markerHeight="12" refX="16" refY="6" viewBox="0 0 16 12" orient="auto" markerUnits="userSpaceOnUse" stroke-dasharray="0,0"><path id="SvgjsPath1035" d="M0,2 L14,6 L0,11 L0,2" fill="#323232" stroke="#323232" stroke-width="2"></path></marker><marker id="SvgjsMarker1038" markerWidth="16" markerHeight="12" refX="16" refY="6" viewBox="0 0 16 12" orient="auto" markerUnits="userSpaceOnUse" stroke-dasharray="0,0"><path id="SvgjsPath1039" d="M0,2 L14,6 L0,11 L0,2" fill="#323232" stroke="#323232" stroke-width="2"></path></marker><marker id="SvgjsMarker1042" markerWidth="16" markerHeight="12" refX="16" refY="6" viewBox="0 0 16 12" orient="auto" markerUnits="userSpaceOnUse" stroke-dasharray="0,0"><path id="SvgjsPath1043" d="M0,2 L14,6 L0,11 L0,2" fill="#323232" stroke="#323232" stroke-width="2"></path></marker><marker id="SvgjsMarker1046" markerWidth="16" markerHeight="12" refX="16" refY="6" viewBox="0 0 16 12" orient="auto" markerUnits="userSpaceOnUse" stroke-dasharray="0,0"><path id="SvgjsPath1047" d="M0,2 L14,6 L0,11 L0,2" fill="#323232" stroke="#323232" stroke-width="2"></path></marker><marker id="SvgjsMarker1050" markerWidth="16" markerHeight="12" refX="16" refY="6" viewBox="0 0 16 12" orient="auto" markerUnits="userSpaceOnUse" stroke-dasharray="0,0"><path id="SvgjsPath1051" d="M0,2 L14,6 L0,11 L0,2" fill="#323232" stroke="#323232" stroke-width="2"></path></marker><marker id="SvgjsMarker1054" markerWidth="16" markerHeight="12" refX="16" refY="6" viewBox="0 0 16 12" orient="auto" markerUnits="userSpaceOnUse" stroke-dasharray="0,0"><path id="SvgjsPath1055" d="M0,2 L14,6 L0,11 L0,2" fill="#323232" stroke="#323232" stroke-width="2"></path></marker><marker id="SvgjsMarker1080" markerWidth="16" markerHeight="12" refX="16" refY="6" viewBox="0 0 16 12" orient="auto" markerUnits="userSpaceOnUse" stroke-dasharray="0,0"><path id="SvgjsPath1081" d="M0,2 L14,6 L0,11 L0,2" fill="#323232" stroke="#323232" stroke-width="2"></path></marker></defs><g id="SvgjsG1008" transform="translate(25,24.999980449676514)"><path id="SvgjsPath1009" d="M 0 35C 0 -11.666666666666666 70 -11.666666666666666 70 35C 70 81.66666666666667 0 81.66666666666667 0 35Z" stroke="rgba(50,50,50,1)" stroke-width="2" fill-opacity="1" fill="#00cc00"></path><g id="SvgjsG1010"><text id="SvgjsText1011" font-family="微软雅黑" text-anchor="middle" font-size="13px" width="50px" fill="#323232" font-weight="400" align="middle" anchor="middle" family="微软雅黑" size="13px" weight="400" font-style="" opacity="1" y="25.55" transform="rotate(0)"><tspan id="SvgjsTspan1012" dy="16" x="35"><tspan id="SvgjsTspan1013" style="text-decoration:;">input</tspan></tspan></text></g></g><g id="SvgjsG1014" transform="translate(267,140.9999804496765)"><path id="SvgjsPath1015" d="M 0 35C 0 -11.666666666666666 70 -11.666666666666666 70 35C 70 81.66666666666667 0 81.66666666666667 0 35Z" stroke="rgba(50,50,50,1)" stroke-width="2" fill-opacity="1" fill="#ffff00"></path><g id="SvgjsG1016"><text id="SvgjsText1017" font-family="微软雅黑" text-anchor="middle" font-size="13px" width="50px" fill="#323232" font-weight="400" align="middle" anchor="middle" family="微软雅黑" size="13px" weight="400" font-style="" opacity="1" y="25.55" transform="rotate(0)"><tspan id="SvgjsTspan1018" dy="16" x="35"><tspan id="SvgjsTspan1019" style="text-decoration:;">node 1</tspan></tspan></text></g></g><g id="SvgjsG1020" transform="translate(25,293.9999804496765)"><path id="SvgjsPath1021" d="M 0 35C 0 -11.666666666666666 70 -11.666666666666666 70 35C 70 81.66666666666667 0 81.66666666666667 0 35Z" stroke="rgba(50,50,50,1)" stroke-width="2" fill-opacity="1" fill="#ffff00"></path><g id="SvgjsG1022"><text id="SvgjsText1023" font-family="微软雅黑" text-anchor="middle" font-size="13px" width="50px" fill="#323232" font-weight="400" align="middle" anchor="middle" family="微软雅黑" size="13px" weight="400" font-style="" opacity="1" y="25.55" transform="rotate(0)"><tspan id="SvgjsTspan1024" dy="16" x="35"><tspan id="SvgjsTspan1025" style="text-decoration:;">node 2</tspan></tspan></text></g></g><g id="SvgjsG1026" transform="translate(267,441.9999804496765)"><path id="SvgjsPath1027" d="M 0 35C 0 -11.666666666666666 70 -11.666666666666666 70 35C 70 81.66666666666667 0 81.66666666666667 0 35Z" stroke="rgba(50,50,50,1)" stroke-width="2" fill-opacity="1" fill="#ffff00"></path><g id="SvgjsG1028"><text id="SvgjsText1029" font-family="微软雅黑" text-anchor="middle" font-size="13px" width="50px" fill="#323232" font-weight="400" align="middle" anchor="middle" family="微软雅黑" size="13px" weight="400" font-style="" opacity="1" y="25.55" transform="rotate(0)"><tspan id="SvgjsTspan1030" dy="16" x="35"><tspan id="SvgjsTspan1031" style="text-decoration:;">node 3</tspan></tspan></text></g></g><g id="SvgjsG1032"><path id="SvgjsPath1033" d="M60 94.99998044967651C 45 172.86665678024292 44 214.86665678024292 60 293.9999804496765" stroke-dasharray="8,5" stroke="#323232" stroke-width="2" fill="none" marker-end="url(#SvgjsMarker1034)"></path></g><g id="SvgjsG1036"><path id="SvgjsPath1037" d="M60 94.99998044967651C 74.7485647305657 263.5768467107039 316.7485647305658 273.42311418864915 302 441.9999804496765" stroke-dasharray="8,5" stroke="#323232" stroke-width="2" fill="none" marker-end="url(#SvgjsMarker1038)"></path></g><g id="SvgjsG1040"><path id="SvgjsPath1041" d="M302 210.9999804496765C 310.0531906298836 303.04837055335383 310.0531906298836 349.9515903459992 302 441.9999804496765" stroke-dasharray="8,5" stroke="#323232" stroke-width="2" fill="none" marker-end="url(#SvgjsMarker1042)"></path></g><g id="SvgjsG1044"><path id="SvgjsPath1045" d="M60 363.9999804496765C 68.86407754147842 465.3168503645069 310.8640775414784 340.68311053484615 302 441.9999804496765" stroke-dasharray="8,5" stroke="#323232" stroke-width="2" fill="none" marker-end="url(#SvgjsMarker1046)"></path></g><g id="SvgjsG1048"><path id="SvgjsPath1049" d="M60 94.99998044967651C 68.58773798957196 193.15827483289445 310.587737989572 42.841686066458635 302 140.9999804496765" stroke-dasharray="8,5" stroke="#323232" stroke-width="2" fill="none" marker-end="url(#SvgjsMarker1050)"></path></g><g id="SvgjsG1052"><path id="SvgjsPath1053" d="M302 210.9999804496765C 310.9190947616536 312.94570006866127 68.91909476165358 192.05426083069176 60 293.9999804496765" stroke-dasharray="8,5" stroke="#323232" stroke-width="2" fill="none" marker-end="url(#SvgjsMarker1054)"></path></g><g id="SvgjsG1056" transform="translate(382,279.8666567802429)"><path id="SvgjsPath1057" d="M 0 26C 0 -8.666666666666666 52 -8.666666666666666 52 26C 52 60.666666666666664 0 60.666666666666664 0 26Z" stroke="rgba(50,50,50,1)" stroke-width="2" fill-opacity="1" fill="#ffff00"></path><g id="SvgjsG1058"><text id="SvgjsText1059" font-family="微软雅黑" text-anchor="middle" font-size="13px" width="32px" fill="#323232" font-weight="400" align="middle" anchor="middle" family="微软雅黑" size="13px" weight="400" font-style="" opacity="1" y="16.55" transform="rotate(0)"></text></g></g><g id="SvgjsG1060" transform="translate(455,295.8666567802429)"><path id="SvgjsPath1061" d="M 0 0L 220 0L 220 20L 0 20Z" stroke="none" fill="none"></path><g id="SvgjsG1062"><text id="SvgjsText1063" font-family="微软雅黑" text-anchor="start" font-size="13px" width="220px" fill="#323232" font-weight="400" align="middle" anchor="start" family="微软雅黑" size="13px" weight="400" font-style="" opacity="1" y="-7.45" transform="rotate(0)"><tspan id="SvgjsTspan1064" dy="16" x="0"><tspan id="SvgjsTspan1065" style="text-decoration:;">choose none/one/multiple input(s) </tspan></tspan><tspan id="SvgjsTspan1066" dy="16" x="0"><tspan id="SvgjsTspan1067" style="text-decoration:;">then add them as output</tspan></tspan></text></g></g><g id="SvgjsG1068" transform="translate(382,210.9999804496765)"><path id="SvgjsPath1069" d="M 0 26C 0 -8.666666666666666 52 -8.666666666666666 52 26C 52 60.666666666666664 0 60.666666666666664 0 26Z" stroke="rgba(50,50,50,1)" stroke-width="2" fill-opacity="1" fill="#00cc00"></path><g id="SvgjsG1070"><text id="SvgjsText1071" font-family="微软雅黑" text-anchor="middle" font-size="13px" width="32px" fill="#323232" font-weight="400" align="middle" anchor="middle" family="微软雅黑" size="13px" weight="400" font-style="" opacity="1" y="16.55" transform="rotate(0)"></text></g></g><g id="SvgjsG1072" transform="translate(455,226.9999804496765)"><path id="SvgjsPath1073" d="M 0 0L 220 0L 220 20L 0 20Z" stroke="none" fill="none"></path><g id="SvgjsG1074"><text id="SvgjsText1075" font-family="微软雅黑" text-anchor="start" font-size="13px" width="220px" fill="#323232" font-weight="400" align="middle" anchor="start" family="微软雅黑" size="13px" weight="400" font-style="" opacity="1" y="0.55" transform="rotate(0)"><tspan id="SvgjsTspan1076" dy="16" x="0"><tspan id="SvgjsTspan1077" style="text-decoration:;">the output of the previous cell</tspan></tspan></text></g></g><g id="SvgjsG1078"><path id="SvgjsPath1079" d="M376 374.8666567802429L404.5 374.8666567802429L404.5 374.8666567802429L433 374.8666567802429" stroke-dasharray="8,5" stroke="#323232" stroke-width="2" fill="none" marker-end="url(#SvgjsMarker1080)"></path></g><g id="SvgjsG1082" transform="translate(463,363.9999804496765)"><path id="SvgjsPath1083" d="M 0 0L 220 0L 220 20L 0 20Z" stroke="none" fill="none"></path><g id="SvgjsG1084"><text id="SvgjsText1085" font-family="微软雅黑" text-anchor="start" font-size="13px" width="220px" fill="#323232" font-weight="400" align="middle" anchor="start" family="微软雅黑" size="13px" weight="400" font-style="" opacity="1" y="-14.95" transform="rotate(0)"><tspan id="SvgjsTspan1086" dy="16" x="0"><tspan id="SvgjsTspan1087" style="text-decoration:;">choose one operation from </tspan></tspan><tspan id="SvgjsTspan1088" dy="16" x="0"><tspan id="SvgjsTspan1089" style="text-decoration:;">MaxPool, AvgPool,, Conv1x1, </tspan></tspan><tspan id="SvgjsTspan1090" dy="16" x="0"><tspan id="SvgjsTspan1091" style="text-decoration:;">Conv3x3, SkipConnect, Zeroize</tspan></tspan></text></g></g></svg>
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment