Resolve conflicts for #4760 (#4762)

docs/_removed/installation.rst

+############
+Installation
+############
+
+Currently we support installation on Linux, Mac and Windows. We also allow you to use docker.
+
+..  toctree::
+    :maxdepth: 2
+
+    Linux & Mac <Tutorial/InstallationLinux>
+    Windows <Tutorial/InstallationWin>
+    Use Docker <Tutorial/HowToUseDocker>
\ No newline at end of file

docs/_removed/reference/python_api.rst

+:orphan:
+
+Python API Reference
+====================
+
+.. autosummary::
+   :toctree: _modules
+   :recursive:
+
+   nni

docs/source/training_services.rst → docs/_removed/training_services.rst

@@ -3,13 +3,9 @@ Introduction to NNI Training Services
 
 ..  toctree::
     Overview <./TrainingService/Overview>
-    Local<./TrainingService/LocalMode>
     Remote<./TrainingService/RemoteMachineMode>
     OpenPAI<./TrainingService/PaiMode>
     Kubeflow<./TrainingService/KubeflowMode>
-    AdaptDL<./TrainingService/AdaptDLMode>
     FrameworkController<./TrainingService/FrameworkControllerMode>
-    DLTS<./TrainingService/DLTSMode>
     AML<./TrainingService/AMLMode>
-    PAI-DLC<./TrainingService/DLCMode>
     Hybrid<./TrainingService/HybridMode>

docs/extension/cardlinkitem.py

@@ -7,6 +7,7 @@ import os
 from docutils.parsers.rst import Directive, directives
 from docutils.statemachine import StringList
 from docutils import nodes
+from sphinx.addnodes import pending_xref
 
 TAG_TEMPLATE = """<span class="card-link-tag">{tag}</span>"""
 
@@ -14,12 +15,12 @@ TAGS_TEMPLATE = """
     <p class="card-link-summary">{tags}</p>
 """
 
-CARD_TEMPLATE = """
+CARD_HEADER = """
 .. raw:: html
 
     <div class="card-link admonition">
 
-    <a href="{link}">
+    <a class="card-link-clickable" href="#">
 
     <div class="card-link-body">
 
@@ -46,6 +47,10 @@ CARD_TEMPLATE = """
     </div>
 
     </a>
+"""
+
+CARD_FOOTER = """
+.. raw:: html
 
     </div>
 """
@@ -95,15 +100,30 @@ class CustomCardItemDirective(Directive):
         else:
             tags_rst = ''
 
-        card_rst = CARD_TEMPLATE.format(header=header,
+        card_rst = CARD_HEADER.format(
+            header=header,
             image=image,
             image_background=image_background,
             link=link,
             description=description,
             tags=tags_rst)
-        card_list = StringList(card_rst.split('\n'))
         card = nodes.paragraph()
-        self.state.nested_parse(card_list, self.content_offset, card)
+        self.state.nested_parse(StringList(card_rst.split('\n')), self.content_offset, card)
+
+        # This needs to corporate with javascript: propagate_card_link.
+        # because sphinx can't correctly handle image in a `pending_xref` after `keepformat`.
+        link_node = pending_xref('<a/>',
+                                 reftype='doc',
+                                 refdomain='std',
+                                 reftarget=link,
+                                 refexplicit=False,
+                                 refwarn=True)
+        link_node += nodes.paragraph(header)
+        link_node['classes'] = ['card-link-anchor']
+        card += link_node
+
+        self.state.nested_parse(StringList(CARD_FOOTER.split('\n')), self.content_offset, card)
+
         return [card]
 
 

docs/extension/codesnippetcard.py

+"""
+Code snippet card, used in index page.
+"""
+
+from docutils.parsers.rst import Directive, directives
+from docutils.statemachine import StringList
+from docutils import nodes
+
+from sphinx.addnodes import pending_xref
+
+
+CARD_TEMPLATE_HEADER = """
+.. raw:: html
+
+    <div class="codesnippet-card admonition">
+
+    <div class="codesnippet-card-body">
+
+    <div class="codesnippet-card-title-container">
+
+    <div class="codesnippet-card-icon">
+
+.. image:: {icon}
+
+.. raw:: html
+
+    </div>
+
+    <h4>{title}</h4>
+    </div>
+
+"""
+
+CARD_TEMPLATE_FOOTER = """
+.. raw:: html
+
+    </div>
+"""
+
+CARD_TEMPLATE_LINK_CONTAINER_HEADER = """
+.. raw:: html
+
+    <div class="codesnippet-card-footer">
+"""
+
+CARD_TEMPLATE_LINK = """
+.. raw:: html
+
+    <div class="codesnippet-card-link">
+    {seemore}
+    <span class="material-icons right">arrow_forward</span>
+    </div>
+"""
+
+
+class CodeSnippetCardDirective(Directive):
+    option_spec = {
+        'icon': directives.unchanged,
+        'title': directives.unchanged,
+        'link': directives.unchanged,
+        'seemore': directives.unchanged,
+    }
+
+    has_content = True
+
+    def run(self):
+        anchor_node = nodes.paragraph()
+
+        try:
+            title = self.options['title']
+            link = directives.uri(self.options['link'])
+            icon = directives.uri(self.options['icon'])
+            seemore = self.options.get('seemore', 'For a full tutorial, please go here.')
+        except ValueError as e:
+            print(e)
+            raise
+
+        # header, title, icon...
+        card_rst = CARD_TEMPLATE_HEADER.format(title=title, icon=icon)
+        card_list = StringList(card_rst.split('\n'))
+        self.state.nested_parse(card_list, self.content_offset, anchor_node)
+
+        # code snippet
+        self.state.nested_parse(self.content, self.content_offset, anchor_node)
+
+        # close body
+        self.state.nested_parse(StringList(CARD_TEMPLATE_FOOTER.split('\n')), self.content_offset, anchor_node)
+
+        # start footer
+        self.state.nested_parse(StringList(CARD_TEMPLATE_LINK_CONTAINER_HEADER.split('\n')), self.content_offset, anchor_node)
+
+        # full tutorial link
+        link_node = pending_xref(CARD_TEMPLATE_LINK,
+                                 reftype='doc',
+                                 refdomain='std',
+                                 reftarget=link,
+                                 refexplicit=False,
+                                 refwarn=True,
+                                 refkeepformat=True)
+        # refkeepformat is handled in `patch_autodoc.py`
+        self.state.nested_parse(StringList(CARD_TEMPLATE_LINK.format(seemore=seemore).split('\n')), self.content_offset, link_node)
+        anchor_node += link_node
+
+        # close footer
+        self.state.nested_parse(StringList(CARD_TEMPLATE_FOOTER.split('\n')), self.content_offset, anchor_node)
+
+        # close whole
+        self.state.nested_parse(StringList(CARD_TEMPLATE_FOOTER.split('\n')), self.content_offset, anchor_node)
+
+        return [anchor_node]
+
+
+def setup(app):
+    app.add_directive('codesnippetcard', CodeSnippetCardDirective)

docs/extension/getpartialtext.py

+"""
+Basically same as
+`sphinx gettext buidler <https://www.sphinx-doc.org/en/master/_modules/sphinx/builders/gettext.html>`_,
+but only get texts from files in a whitelist.
+"""
+
+import re
+
+from docutils import nodes
+from sphinx.application import Sphinx
+from sphinx.builders.gettext import MessageCatalogBuilder
+
+
+class PartialMessageCatalogBuilder(MessageCatalogBuilder):
+    name = 'getpartialtext'
+
+    def init(self):
+        super().init()
+
+        self.whitelist_docs = [re.compile(x) for x in self.config.gettext_documents]
+
+    def write_doc(self, docname: str, doctree: nodes.document) -> None:
+        for doc_re in self.whitelist_docs:
+            if doc_re.match(docname):
+                return super().write_doc(docname, doctree)
+
+
+def setup(app: Sphinx):
+    app.add_builder(PartialMessageCatalogBuilder)
+    app.add_config_value('gettext_documents', [], 'gettext')

docs/extension/patch_autodoc.py

+"""Hack autodoc to get more fine-grained docstring rendering contol.
+
+autodoc and autosummary didn't expose many of their controls to sphinx users via config.
+To customize them, the "correct" approach seems to copy and paste all their code and rewrite some part.
+To avoid doing this, I monkey-patched some of the functions to keep the changes minimal.
+
+Note that some of them are related to sphinx internal APIs, which can be broken when sphinx got upgraded.
+Try to keep them updated, or pin to a particular sphinx version.
+"""
+
+import inspect
+import os
+from typing import List, Tuple, List
+
+import sphinx
+from docutils import nodes
+from docutils.nodes import Node
+
+
+class ClassNewBlacklistPatch:
+    """Force some classes to skip ``__new__`` when generating signature."""
+
+    original = None
+
+    def restore(self, *args, **kwargs):
+        assert self.original is not None
+        sphinx.ext.autodoc._CLASS_NEW_BLACKLIST = self.original
+
+    def patch(self, *args, **kwargs):
+        self.original = sphinx.ext.autodoc._CLASS_NEW_BLACKLIST
+
+        blacklist = []
+
+        import nni.retiarii.nn.pytorch
+        for name in dir(nni.retiarii.nn.pytorch):
+            obj = getattr(nni.retiarii.nn.pytorch, name)
+            if inspect.isclass(obj):
+                new_name = "{0.__module__}.{0.__qualname__}".format(obj.__new__)
+                if new_name not in blacklist:
+                    blacklist.append(new_name)
+
+        sphinx.ext.autodoc._CLASS_NEW_BLACKLIST = self.original + blacklist
+
+
+def disable_trace_patch(*args, **kwargs):
+    """Disable trace by setting an environment variable."""
+    os.environ['NNI_TRACE_FLAG'] = 'DISABLE'
+
+
+def trial_tool_import_patch(*args, **kwargs):
+    """Insert dummy trial tool variable to ensure trial_tool can be imported.
+    See nni/tools/trial_tool/constants.py
+    """
+    os.environ.update({
+        'NNI_OUTPUT_DIR': '/tmp',
+        'NNI_PLATFORM': 'unittest',
+        'NNI_SYS_DIR': '/tmp',
+        'NNI_TRIAL_JOB_ID': 'dummy',
+        'NNI_EXP_ID': 'dummy',
+        'MULTI_PHASE': 'dummy'
+    })
+
+
+class AutoSummaryPatch:
+    """Ignore certain files as they are completely un-importable. It patches:
+
+    - find_autosummary_in_files: Some modules cannot be imported at all due to dependency issues or some special design.
+      They need to skipped when running autosummary generate.
+    - Autosummary.get_table: The original autosummary creates an index for each module, and the module links in autosummary table
+      points to the corresponding generated module page (by using ``:py:module:xxx``). This doesn't work for us,
+      because we have used automodule else (other than autosummary) in our docs, and to avoid duplicate index,
+      we have to set ``:noindex:`` in autosummary template (see docs/templates/autosummary/module.rst).
+      This breaks most of the links, where they fail to link to generated module page by using index.
+      We here update the python domain role, to a general domain role (``:doc:``), and link to the page directly.
+    """
+
+    find_autosummary_original = None
+    get_table_original = None
+
+    def restore(self, *args, **kwargs):
+        assert self.find_autosummary_original is not None and self.get_table_original is not None
+        sphinx.ext.autosummary.generate.find_autosummary_in_files = self.find_autosummary_original
+        sphinx.ext.autosummary.Autosummary.get_table = self.get_table_original
+
+    def patch(self, app, config):
+        from sphinx.ext.autosummary import Autosummary
+        from sphinx.ext.autosummary.generate import AutosummaryEntry
+
+        self.find_autosummary_original = sphinx.ext.autosummary.generate.find_autosummary_in_files
+        self.get_table_original = Autosummary.get_table
+
+        def find_autosummary_in_files(filenames: List[str]) -> List[AutosummaryEntry]:
+            items: List[AutosummaryEntry] = self.find_autosummary_original(filenames)
+            items = [item for item in items if item.name not in config.autosummary_mock_imports]
+            return items
+
+        def get_table(autosummary, items: List[Tuple[str, str, str, str]]) -> List[Node]:
+            col_spec, autosummary_table = self.get_table_original(autosummary, items)
+            if 'toctree' in autosummary.options:
+                # probably within modules
+                table = autosummary_table[0]
+                tgroup = table[0]
+                tbody = tgroup[-1]
+                for row in tbody:
+                    entry = row[0]
+                    paragraph = entry[0]
+                    pending_xref = paragraph[0]
+
+                    # get the reference path and check whether it has been generated
+                    # if path to reference is changed, this should also be changed
+                    reftarget_path = 'reference/_modules/' + pending_xref['reftarget']
+
+                    if reftarget_path in autosummary.env.found_docs:
+                        # make :py:obj:`xxx` looks like a :doc:`xxx`
+                        pending_xref['refdomain'] = 'std'
+                        pending_xref['reftype'] = 'doc'
+                        pending_xref['refexplicit'] = False
+                        pending_xref['refwarn'] = True
+                        pending_xref['reftarget'] = '/' + reftarget_path
+                        # a special tag to enable `ResolveDocPatch`
+                        pending_xref['refkeepformat'] = True
+
+            return [col_spec, autosummary_table]
+
+        sphinx.ext.autosummary.generate.find_autosummary_in_files = find_autosummary_in_files
+        sphinx.ext.autosummary.Autosummary.get_table = get_table
+
+
+class ResolveDocPatch:
+    """Original :doc: role throws away all the format, and keep raw text only.
+    We wish to keep module names literal. This patch is to keep literal format in :doc: resolver."""
+
+    original = None
+
+    def restore(self, *args, **kwargs):
+        assert self.original is not None
+        sphinx.domains.std.StandardDomain._resolve_doc_xref = self.original
+
+    def patch(self, *args, **kwargs):
+        self.original = sphinx.domains.std.StandardDomain._resolve_doc_xref
+
+        def doc_xref_resolver(std_domain, env, fromdocname, builder, typ, target, node, contnode):
+            if not node.get('refkeepformat'):
+                # redirect to original implementation to make it safer
+                return self.original(std_domain, env, fromdocname, builder, typ, target, node, contnode)
+
+            # directly reference to document by source name; can be absolute or relative
+            from sphinx.domains.std import docname_join, make_refnode
+            refdoc = node.get('refdoc', fromdocname)
+            docname = docname_join(refdoc, node['reftarget'])
+            if docname not in env.all_docs:
+                return None
+            else:
+                innernode = node[0]  # no astext here, to keep literal intact
+                return make_refnode(builder, fromdocname, docname, None, innernode)
+
+        sphinx.domains.std.StandardDomain._resolve_doc_xref = doc_xref_resolver
+
+
+def setup(app):
+    # See life-cycle of sphinx app here:
+    # https://www.sphinx-doc.org/en/master/extdev/appapi.html#sphinx-core-events
+    patch = ClassNewBlacklistPatch()
+    app.connect('env-before-read-docs', patch.patch)
+    app.connect('env-merge-info', patch.restore)
+
+    patch = ResolveDocPatch()
+    app.connect('env-before-read-docs', patch.patch)
+    app.connect('env-merge-info', patch.restore)
+
+    app.connect('env-before-read-docs', disable_trace_patch)
+
+    # autosummary generate happens at builder-inited
+    app.connect('config-inited', trial_tool_import_patch)
+
+    autosummary_patch = AutoSummaryPatch()
+    app.connect('config-inited', autosummary_patch.patch)
+    app.connect('env-merge-info', autosummary_patch.restore)

docs/extension/patch_docutils.py

-"""Additional docutils patch to suppress warnings in i18n documentation build."""
-
-from typing import Any
-
-import docutils
-from docutils.utils import Reporter
-
-
-class Patch:
-    """
-    This is actually done in sphinx, but sphinx didn't replace all `get_language` occurrences.
-    https://github.com/sphinx-doc/sphinx/blob/680417a10df7e5c35c0ff65979bd22906b9a5f1e/sphinx/util/docutils.py#L127
-
-    Related issue:
-    https://github.com/sphinx-doc/sphinx/issues/10179
-    """
-
-    original = None
-
-    def restore(self, *args, **kwargs):
-        assert self.original is not None
-        docutils.parsers.rst.languages.get_language = self.original
-
-    def patch(self, *args, **kwargs):
-        from docutils.parsers.rst.languages import get_language
-        self.original = get_language
-
-        def patched_get_language(language_code: str, reporter: Reporter = None) -> Any:
-            return get_language(language_code)
-
-        docutils.parsers.rst.languages.get_language = patched_get_language
-
-
-def setup(app):
-    # See life-cycle of sphinx app here:
-    # https://www.sphinx-doc.org/en/master/extdev/appapi.html#sphinx-core-events
-    patch = Patch()
-    app.connect('env-before-read-docs', patch.patch)
-    app.connect('env-merge-info', patch.restore)

docs/extension/toctree_check.py

+"""
+Make sure pages that contain toctree only has a toctree,
+because, if our theme is used, other contents will not be visible.
+"""
+
+import re
+
+from docutils import nodes
+from sphinx.application import Sphinx
+from sphinx.addnodes import toctree
+from sphinx.util.logging import getLogger
+
+logger = getLogger('toctree_check')
+
+def _strip_compound(node):
+    if isinstance(node, nodes.compound):
+        return _strip_compound(node[0])
+    return node
+
+
+def toctree_check(app: Sphinx, doctree: nodes.document, docname: str):
+    whitelist = app.config.toctree_check_whitelist
+
+    if docname in whitelist:
+        return
+
+    # Scan top-level nodes
+    has_toctree = False
+
+    other_types = []
+
+    for i in range(len(doctree[0])):
+        node = doctree[0][i]
+        if isinstance(_strip_compound(node), toctree):
+            has_toctree = True
+        elif isinstance(_strip_compound(node), nodes.title):
+            # Allow title
+            pass
+        else:
+            other_types.append(type(_strip_compound(node)))
+
+    if has_toctree and other_types:
+        # We don't allow a document with toctree to have other types of contents
+        logger.warning('Expect a toctree document to contain only a toctree, '
+                       'but found other types of contents: %s', str(set(other_types)),
+                       location=docname)
+
+
+def setup(app):
+    app.connect('doctree-resolved', toctree_check)
+
+    app.add_config_value('toctree_check_whitelist', [], True)

docs/extension/tutorial_links.py

+"""Creating hard links for tutorials in each individual topics."""
+
+import os
+import re
+
+HEADER = """.. THIS FILE IS A COPY OF {} WITH MODIFICATIONS.
+.. TO MAKE ONE TUTORIAL APPEAR IN MULTIPLE PLACES.
+
+"""
+
+def flatten_filename(filename):
+    return filename.replace('/', '_').replace('.', '_')
+
+def copy_tutorials(app):
+    # TODO: use sphinx logger
+    print('[tutorial links] copy tutorials...')
+    for src, tar in app.config.tutorials_copy_list:
+        target_path = os.path.join(app.srcdir, tar)
+        content = open(os.path.join(app.srcdir, src)).read()
+
+        # Add a header
+        content = HEADER.format(src) + content
+
+        # Add a prefix to labels to avoid duplicates.
+        label_map = {}
+
+        # find all anchors:   https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html
+        # but not hyperlinks: https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html#external-links
+        for prefix, label_name in list(re.findall(r'(\.\.\s*_)(.*?)\:\s*\n', content)):
+            label_map[label_name] = flatten_filename(tar) + '_' + label_name
+            # anchor
+            content = content.replace(prefix + label_name + ':', prefix + label_map[label_name] + ':')
+            # :ref:`xxx`
+            content = content.replace(f':ref:`{label_name}`', f':ref:`{label_map[label_name]}')
+            # :ref:`yyy <xxx>`
+            content = re.sub(r"(\:ref\:`.*?\<)" + label_name + r"(\>`)", r'\1' + label_map[label_name] + r'\2', content)
+
+        open(target_path, 'w').write(content)
+
+
+def setup(app):
+    # See life-cycle of sphinx app here:
+    # https://www.sphinx-doc.org/en/master/extdev/appapi.html#sphinx-core-events
+    app.connect('builder-inited', copy_tutorials)
+    app.add_config_value('tutorials_copy_list', [], True, [list])