Bootstrapping tutorials in documentation (#4522)

.gitattributes

+# The following types are treated as text
+# EOL are converted to LF
+*.tsx   text    eol=lf
 *.ts    text    eol=lf
 *.py    text    eol=lf
 *.sh    text    eol=lf
+
+# The following types are treated as binary
+# These files appear mostly in docs/ folder
+.ipynb  binary
+.pickle binary
+
+# Linguist override (for github)
+docs/source/tutorials/*     linguist-generated

.readthedocs.yaml

+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-20.04
+  tools:
+    python: "3.9"
+    nodejs: "16"  # specified but actually not used
+    # You can also specify other tool versions:
+    # rust: "1.55"
+    # golang: "1.17"
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/source/conf.py
+
+# Optionally declare the Python requirements required to build your docs
+python:
+  install:
+  - requirements: dependencies/develop.txt
+  - requirements: dependencies/required.txt
+  # The issue with smac and swig prevents us from installing required_extra.
+  # As a result, the docstring from several tuners including SMAC, PPO cannot be rendered.
+  # - requirements: dependencies/required_extra.txt
+  - requirements: dependencies/recommended.txt
+  # We cannot have `python setup.py install` here,
+  # because it's not supported by NNI.

dependencies/develop.txt

@@ -8,8 +8,8 @@ pylint
 pytest
 pytest-azurepipelines
 pytest-cov
+rstcheck
 sphinx
 sphinx-argparse
-sphinx-rtd-theme
-sphinxcontrib-websupport
+sphinx-gallery
 git+https://github.com/bashtage/sphinx-material.git

docs/.gitignore

-**/_build
+build/
+
+# legacy build
+_build/

docs/en_US/Makefile → docs/Makefile

@@ -4,8 +4,8 @@
 # You can set these variables from the command line.
 SPHINXOPTS    =
 SPHINXBUILD   = sphinx-build
-SOURCEDIR     = .
-BUILDDIR      = _build
+SOURCEDIR     = source
+BUILDDIR      = build
 
 # Put it first so that "make" without argument is like "make help".
 help:

docs/en_US/builtin_tuner.rst

-Builtin-Tuners
-==============
-
-NNI provides an easy way to adopt an approach to set up parameter tuning algorithms, we call them **Tuner**.
-
-Tuner receives metrics from `Trial` to evaluate the performance of a specific parameters/architecture configuration. Tuner sends the next hyper-parameter or architecture configuration to Trial.
-
-
-..  toctree::
-    :maxdepth: 1
-    
-    Overview <Tuner/BuiltinTuner>
-    TPE <Tuner/TpeTuner>
-    Random Search <Tuner/RandomTuner>
-    Anneal <Tuner/AnnealTuner>
-    Naive Evolution <Tuner/EvolutionTuner>
-    SMAC <Tuner/SmacTuner>
-    Metis Tuner <Tuner/MetisTuner>
-    Batch Tuner <Tuner/BatchTuner>
-    Grid Search <Tuner/GridsearchTuner>
-    GP Tuner <Tuner/GPTuner>
-    Network Morphism <Tuner/NetworkmorphismTuner>
-    Hyperband <Tuner/HyperbandAdvisor>
-    BOHB <Tuner/BohbAdvisor>
-    PBT Tuner <Tuner/PBTTuner>
-    DNGO Tuner <Tuner/DngoTuner>

docs/extension/cardlinkitem.py

+"""
+Directive "cardlinkitem" used in tutorials navigation page.
+"""
+
+import os
+
+from docutils.parsers.rst import Directive, directives
+from docutils.statemachine import StringList
+from docutils import nodes
+
+TAG_TEMPLATE = """<span class="card-link-tag">{tag}</span>"""
+
+TAGS_TEMPLATE = """
+    <p class="card-link-summary">{tags}</p>
+"""
+
+CARD_TEMPLATE = """
+.. raw:: html
+
+    <div class="card-link admonition">
+
+    <a href="{link}">
+
+    <div class="card-link-body">
+
+    <div class="card-link-text">
+
+    <div class="card-link-title-container">
+        <h4>{header}</h4>
+    </div>
+
+    <p class="card-link-summary">{description}</p>
+
+    {tags}
+
+    </div>
+
+    <div class="card-link-icon circle {image_background}">
+
+.. image:: {image}
+
+.. raw:: html
+
+    </div>
+
+    </div>
+
+    </a>
+
+    </div>
+"""
+
+
+class CustomCardItemDirective(Directive):
+    option_spec = {
+        'header': directives.unchanged,
+        'image': directives.unchanged,
+        'background': directives.unchanged,
+        'link': directives.unchanged,
+        'description': directives.unchanged,
+        'tags': directives.unchanged
+    }
+
+    def run(self):
+        env = self.state.document.settings.env
+
+        try:
+            if 'header' in self.options:
+                header = self.options['header']
+            else:
+                raise ValueError('header not found')
+
+            if 'link' in self.options:
+                link = directives.uri(self.options['link'])
+            else:
+                raise ValueError('link not found')
+
+            if 'image' in self.options:
+                image = directives.uri(self.options['image'])
+            else:
+                image = os.path.join(os.path.relpath(env.app.srcdir, env.app.confdir), '../img/thumbnails/nni_icon_white.png')
+
+            image_background = self.options.get('background', 'indigo')
+            description = self.options.get('description', '')
+
+            tags = self.options.get('tags', '').strip().split('/')
+            tags = [t.strip() for t in tags if t.strip()]
+
+        except ValueError as e:
+            print(e)
+            raise
+
+        if tags:
+            tags_rst = TAGS_TEMPLATE.format(tags=''.join([TAG_TEMPLATE.format(tag=tag) for tag in tags]))
+        else:
+            tags_rst = ''
+
+        card_rst = CARD_TEMPLATE.format(header=header,
+                                        image=image,
+                                        image_background=image_background,
+                                        link=link,
+                                        description=description,
+                                        tags=tags_rst)
+        card_list = StringList(card_rst.split('\n'))
+        card = nodes.paragraph()
+        self.state.nested_parse(card_list, self.content_offset, card)
+        return [card]
+
+
+def setup(app):
+    app.add_directive('cardlinkitem', CustomCardItemDirective)

docs/extension/inplace_translation.py

+"""
+Sphinx inplace translation.
+Please put `xxx_zh_CN.rst` alongside `xxx.rst`. When language is set to `zh_CN`,
+`xxx_zh_CN.rst` will be used in place of `xxx.rst`.
+If translation does not exist, it will automatically fallback to the original files, without warning.
+
+I write this based on the example of:
+https://github.com/readthedocs/sphinxcontrib-multisrc/blob/master/sphinxcontrib/multisrc.py
+"""
+
+import os
+import types
+
+
+def builder_inited(app):
+    """Event listener to set up multiple environments."""
+    patch_doc2path(app.env, app.config.language)
+
+
+def patch_doc2path(env, language):
+    # patch doc2path so that it resolves to the correct language.
+    override_doc2path = env.doc2path
+
+    def doc2path(env, docname: str, base: bool = True):
+        path = override_doc2path(docname, base)
+        if language not in (None, 'en'):
+            # The language is set to another one
+            new_docname = f'{docname}_{language}'
+            new_path = override_doc2path(new_docname, base)
+            if os.path.exists(new_path):
+                return new_path
+        return path
+
+    env.doc2path = types.MethodType(doc2path, env)
+
+
+def setup(app):
+    app.connect('builder-inited', builder_inited)

docs/extension/patch_docutils.py

+"""Additional docutils patch to suppress warnings in i18n documentation build."""
+
+from typing import Any
+
+import docutils
+from docutils.utils import Reporter
+
+
+class Patch:
+    """
+    This is actually done in sphinx, but sphinx didn't replace all `get_language` occurrences.
+    https://github.com/sphinx-doc/sphinx/blob/680417a10df7e5c35c0ff65979bd22906b9a5f1e/sphinx/util/docutils.py#L127
+
+    Related issue:
+    https://github.com/sphinx-doc/sphinx/issues/10179
+    """
+
+    original = None
+
+    def restore(self, *args, **kwargs):
+        assert self.original is not None
+        docutils.parsers.rst.languages.get_language = self.original
+
+    def patch(self, *args, **kwargs):
+        from docutils.parsers.rst.languages import get_language
+        self.original = get_language
+
+        def patched_get_language(language_code: str, reporter: Reporter = None) -> Any:
+            return get_language(language_code)
+
+        docutils.parsers.rst.languages.get_language = patched_get_language
+
+
+def setup(app):
+    # See life-cycle of sphinx app here:
+    # https://www.sphinx-doc.org/en/master/extdev/appapi.html#sphinx-core-events
+    patch = Patch()
+    app.connect('env-before-read-docs', patch.patch)
+    app.connect('env-merge-info', patch.restore)

docs/requirements.txt

-sphinx>=4.0
-sphinx-argparse
-sphinx-rtd-theme
-sphinxcontrib-websupport
-pygments>=2.7.1
-hyperopt
-json_tricks
-numpy
-scipy
-coverage
-peewee
-nbsphinx
-schema
-tensorboard
-scikit-learn>=0.24.1
-thop
-colorama
-pkginfo
-websockets
-filelock
-prettytable
-psutil
-pyyaml
-ipython
-gym
-tianshou
-https://download.pytorch.org/whl/cpu/torch-1.7.1%2Bcpu-cp37-cp37m-linux_x86_64.whl
-https://download.pytorch.org/whl/cpu/torchvision-0.8.2%2Bcpu-cp37-cp37m-linux_x86_64.whl
-pytorch-lightning
-onnx
-git+https://github.com/bashtage/sphinx-material.git