Commit 1011377c authored by qianyj's avatar qianyj
Browse files

the source code of NNI for DCU

parent abc22158
"""
This is to keep Chinese doc update to English doc. Should be run regularly.
The files in whitelist will be kept unchanged, as they will be translated manually.
Under docs, run
python tools/chineselink.py
"""
import hashlib
import os
import shutil
import sys
from pathlib import Path
def walk(path):
for p in Path(path).iterdir():
if p.is_dir():
yield from walk(p)
continue
yield p
# Keeps files as discussed in
# https://github.com/microsoft/nni/issues/4298
# Not the recommended way of sphinx though: https://docs.readthedocs.io/en/stable/guides/manage-translations-sphinx.html
whitelist = [
'_templates/index.html', # I think no one ever remembers to update this file. Might need to rethink about this.
'Overview.rst',
'installation.rst',
'Tutorial/InstallationLinux.rst',
'Tutorial/InstallationWin.rst',
'Tutorial/QuickStart.rst',
'TrialExample/Trials.rst',
'Tutorial/WebUI.rst',
'NAS/QuickStart.rst',
'Compression/Overview.rst',
'Compression/QuickStart.rst',
]
suffix_list = [
'.html',
'.md',
'.rst',
'.ipynb',
]
for path in whitelist:
assert (Path('zh_CN') / path).exists(), path
content_tables = []
for path in walk(Path('en_US')):
if path.suffix == '.rst':
is_content_table = False
for line in path.open('r').readlines():
if is_content_table:
if not line.startswith(' ') and line.strip():
is_content_table = False
if 'toctree::' in line:
is_content_table = True
if is_content_table:
content_tables.append(path.relative_to('en_US').as_posix())
print('Whitelist:' ,content_tables)
whitelist += content_tables
pipeline_mode = len(sys.argv) > 1 and sys.argv[1] == 'check'
failed_files = []
def need_to_translate(source, target):
if not target.exists():
failed_files.append('(missing) ' + target.as_posix())
if pipeline_mode:
return
shutil.copyfile(source, target)
if target.suffix == '.html':
return # FIXME I don't know how to process html
target_checksum = hashlib.sha256(path.open('rb').read()).hexdigest()[:32]
checksum = target.open('r').readline().strip()[3:]
if checksum != target_checksum:
failed_files.append('(out-of-date) ' + target.as_posix())
if pipeline_mode:
return
contents = target.open('r').readlines()
firstline = '.. ' + target_checksum + '\n'
if contents[0].startswith('.. '):
contents = [firstline] + contents[1:]
else:
contents = [firstline, '\n'] + contents
target.open('w').writelines(contents)
for path in walk(Path('en_US')):
relative_path = path.relative_to('en_US')
if relative_path.as_posix().startswith('_build'):
continue
if path.suffix in suffix_list:
target_path = (Path('zh_CN') / relative_path)
if relative_path.as_posix() in whitelist:
# whitelist files. should be translated
need_to_translate(path, target_path)
print(f'Skipped linking for {path} as it is in whitelist.')
else:
target_path.parent.mkdir(exist_ok=True)
link_path = path
for _ in range(len(list(Path(relative_path).parents))):
link_path = Path('..') / link_path
if not target_path.is_symlink() or os.readlink(target_path) != link_path.as_posix():
failed_files.append('(invalid link) ' + target_path.as_posix())
if not pipeline_mode:
target_path.unlink(missing_ok=True)
target_path.symlink_to(link_path)
# delete redundant files
for path in walk(Path('zh_CN')):
if path.suffix in suffix_list:
relative_path = path.relative_to('zh_CN')
if not (Path('en_US') / relative_path).exists():
failed_files.append('(redundant) ' + path.as_posix())
if not pipeline_mode:
print(f'Deleting {path}')
path.unlink()
if pipeline_mode and failed_files:
raise ValueError(
'The following files are not up-to-date. Please run "python3 tools/chineselink.py" under docs folder '
'to refresh them and update their corresponding translation.\n' + '\n'.join([' ' + line for line in failed_files]))
if failed_files:
print('Updated files:', failed_files)
import argparse
import m2r
import os
import re
import shutil
from pathlib import Path
def single_line_process(line):
if line == ' .. contents::':
return '.. contents::'
# https://github.com/sphinx-doc/sphinx/issues/3921
line = re.sub(r'(`.*? <.*?>`)_', r'\1__', line)
# inline emphasis
line = re.sub(r'\*\*\\ (.*?)\\ \*\*', r' **\1** ', line)
line = re.sub(r'\*(.*?)\\ \*', r'*\1*', line)
line = re.sub(r'\*\*(.*?) \*\*', r'**\1** ', line)
line = re.sub(r'\\\*\\\*(.*?)\*\*', r'**\1**', line)
line = re.sub(r'\\\*\\\*(.*?)\*\*\\ ', r'**\1**', line)
line = line.replace(r'\* - `\**', r'* - `**')
line = re.sub(r'\\\* \*\*(.*?)\*\* \(\\\*\s*(.*?)\s*\*\\ \)', r'* \1 (\2)', line)
line = re.sub(r'\<(.*)\.md(\>|#)', r'<\1.rst\2', line)
line = re.sub(r'`\*\*(.*?)\*\* <#(.*?)>`__', r'`\1 <#\2>`__', line)
line = re.sub(r'\*\* (classArgs|stop|FLOPS.*?|pruned.*?|large.*?|path|pythonPath|2D.*?|codeDirectory|ps|worker|Tuner|Assessor)\*\*',
r' **\1**', line)
line = line.replace('.. code-block:::: bash', '.. code-block:: bash')
line = line.replace('raw-html-m2r', 'raw-html')
line = line.replace('[toc]', '.. toctree::')
# image
line = re.sub(r'\:raw\-html\:`\<img src\=\"(.*?)\" style\=\"zoom\: ?(\d+)\%\;\" \/\>`', r'\n.. image:: \1\n :scale: \2%', line)
# special case (per line handling)
line = line.replace('Nb = |Db|', r'Nb = \|Db\|')
line = line.replace(' Here is just a small list of libraries ', '\nHere is just a small list of libraries ')
line = line.replace(' Find the data management region in job submission page.', 'Find the data management region in job submission page.')
line = line.replace('Tuner/InstallCustomizedTuner.md', 'Tuner/InstallCustomizedTuner')
line = line.replace('&#10003;', ':raw-html:`&#10003;`')
line = line.replace(' **builtinTunerName** and** classArgs**', '**builtinTunerName** and **classArgs**')
line = line.replace('`\ ``nnictl ss_gen`` <../Tutorial/Nnictl.rst>`__', '`nnictl ss_gen <../Tutorial/Nnictl.rst>`__')
line = line.replace('**Step 1. Install NNI, follow the install guide `here <../Tutorial/QuickStart.rst>`__.**',
'**Step 1. Install NNI, follow the install guide** `here <../Tutorial/QuickStart.rst>`__.')
line = line.replace('*Please refer to `here ', 'Please refer to `here ')
# line = line.replace('\* **optimize_mode** ', '* **optimize_mode** ')
if line == '~' * len(line):
line = '^' * len(line)
return line
def special_case_replace(full_text):
replace_pairs = {}
replace_pairs['PyTorch\n"""""""'] = '**PyTorch**'
replace_pairs['Search Space\n============'] = '.. role:: raw-html(raw)\n :format: html\n\nSearch Space\n============'
for file in os.listdir(Path(__file__).parent / 'patches'):
with open(Path(__file__).parent / 'patches' / file) as f:
r, s = f.read().split('%%%%%%\n')
replace_pairs[r] = s
for r, s in replace_pairs.items():
full_text = full_text.replace(r, s)
return full_text
def process_table(content):
content = content.replace('------ |', '------|')
lines = []
for line in content.split('\n'):
if line.startswith(' |'):
line = line[2:]
lines.append(line)
return '\n'.join(lines)
def process_github_link(line):
line = re.sub(r'`(\\ ``)?([^`]*?)(``)? \<(.*?)(blob|tree)/v1.9/(.*?)\>`__', r':githublink:`\2 <\6>`', line)
if 'githublink' in line:
line = re.sub(r'\*Example: (.*)\*', r'*Example:* \1', line)
line = line.replace('https://nni.readthedocs.io/en/latest', '')
return line
for root, dirs, files in os.walk('en_US'):
root = Path(root)
for file in files:
if not file.endswith('.md') or file == 'Release_v1.0.md':
continue
with open(root / file) as f:
md_content = f.read()
if file == 'Nnictl.md':
md_content = process_table(md_content)
out = m2r.convert(md_content)
lines = out.split('\n')
if lines[0] == '':
lines = lines[1:]
# remove code-block eval_rst
i = 0
while i < len(lines):
line = lines[i]
if line.strip() == '.. code-block:: eval_rst':
space_count = line.index('.')
lines[i] = lines[i + 1] = None
if i > 0 and lines[i - 1]:
lines[i] = '' # blank line
i += 2
while i < len(lines) and (lines[i].startswith(' ' * (space_count + 3)) or lines[i] == ''):
lines[i] = lines[i][space_count + 3:]
i += 1
elif line.strip() == '.. code-block' or line.strip() == '.. code-block::':
lines[i] += ':: bash'
i += 1
else:
i += 1
lines = [l for l in lines if l is not None]
lines = list(map(single_line_process, lines))
if file != 'Release.md':
# githublink
lines = list(map(process_github_link, lines))
out = '\n'.join(lines)
out = special_case_replace(out)
with open(root / (Path(file).stem + '.rst'), 'w') as f:
f.write(out)
# back it up and remove
moved_root = Path('archive_en_US') / root.relative_to('en_US')
moved_root.mkdir(exist_ok=True)
shutil.move(root / file, moved_root / file)
* - GP Tuner
- :raw-html:`&#10003;`
-
- :raw-html:`&#10003;`
- :raw-html:`&#10003;`
- :raw-html:`&#10003;`
- :raw-html:`&#10003;`
- :raw-html:`&#10003;`
-
-
-
%%%%%%
* - GP Tuner
- :raw-html:`&#10003;`
-
- :raw-html:`&#10003;`
- :raw-html:`&#10003;`
- :raw-html:`&#10003;`
- :raw-html:`&#10003;`
- :raw-html:`&#10003;`
-
-
-
-
An SSH server needs a port; you need to expose Docker's SSH port to NNI as the connection port. For example, if you set your container's SSH port as **``A``** \ , you should map the container's port ** ``A``** to your remote host machine's other port ** ``B``** \ , NNI will connect port ** ``B``** as an SSH port, and your host machine will map the connection from port ** ``B``** to port ** ``A``** then NNI could connect to your Docker container.
%%%%%%
An SSH server needs a port; you need to expose Docker's SSH port to NNI as the connection port. For example, if you set your container's SSH port as ``A``, you should map the container's port ``A`` to your remote host machine's other port ``B``, NNI will connect port ``B`` as an SSH port, and your host machine will map the connection from port ``B`` to port ``A`` then NNI could connect to your Docker container.
If the id ends with *, nnictl will stop all experiments whose ids matchs the regular.
%%%%%%
If the id ends with \*, nnictl will stop all experiments whose ids matchs the regular.
..
make: *** [install-XXX] Segmentation fault (core dumped)
%%%%%%
.. code-block:: text
make: *** [install-XXX] Segmentation fault (core dumped)
Click ``Submit job`` button in web portal.
%%%%%%
Click ``Submit job`` button in web portal.
:raw-html:`<div >
<img src="https://github.com/microsoft/Cream/blob/main/demo/intro.jpg" width="800"/>
</div>`
%%%%%%
:raw-html:`<div ><img src="https://github.com/microsoft/Cream/blob/main/demo/intro.jpg" width="800"/></div>`
\ No newline at end of file
.. list-table::
:header-rows: 1
%%%%%%
.. list-table::
:header-rows: 1
:widths: auto
.. code-block:: bash
1.1 Declare NNI API
Include `import nni` in your trial code to use NNI APIs.
%%%%%%
..
1.1 Declare NNI API
Include `import nni` in your trial code to use NNI APIs.
.. code-block:: bash
from nni.compression.pytorch.utils.counter import count_flops_params
%%%%%%
.. code-block:: python
from nni.compression.pytorch.utils.counter import count_flops_params
.. code-block:: bash
NNI's official image msranni/nni does not support SSH servers for the time being; you should build your own Docker image with an SSH configuration or use other images as a remote server.
%%%%%%
.. code-block:: text
NNI's official image msranni/nni does not support SSH servers for the time being; you should build your own Docker image with an SSH configuration or use other images as a remote server.
Code Styles & Naming Conventions
--------------------------------
* We follow `PEP8 <https://www.python.org/dev/peps/pep-0008/>`__ for Python code and naming conventions, do try to adhere to the same when making a pull request or making a change. One can also take the help of linters such as ``flake8`` or ``pylint``
* We also follow `NumPy Docstring Style <https://www.sphinx-doc.org/en/master/usage/extensions/example_numpy.html#example-numpy>`__ for Python Docstring Conventions. During the `documentation building <Contributing.rst#documentation>`__\ , we use `sphinx.ext.napoleon <https://www.sphinx-doc.org/en/master/usage/extensions/napoleon.html>`__ to generate Python API documentation from Docstring.
* For docstrings, please refer to `numpydoc docstring guide <https://numpydoc.readthedocs.io/en/latest/format.html>`__ and `pandas docstring guide <https://python-sprints.github.io/pandas/guide/pandas_docstring.html>`__
* For function docstring, **description** , **Parameters**\ , and** Returns**\ /** Yields** are mandatory.
* For class docstring, **description**\ ,** Attributes** are mandatory.
* For docstring to describe ``dict``\ , which is commonly used in our hyper-param format description, please refer to [RiboKit : Doc Standards
* Internal Guideline on Writing Standards](https://ribokit.github.io/docs/text/)
Documentation
-------------
Our documentation is built with :githublink:`sphinx <docs>`.
*
Before submitting the documentation change, please **build homepage locally**\ : ``cd docs/en_US && make html``\ , then you can see all the built documentation webpage under the folder ``docs/en_US/_build/html``. It's also highly recommended taking care of** every WARNING** during the build, which is very likely the signal of a** deadlink** and other annoying issues.
*
For links, please consider using **relative paths** first. However, if the documentation is written in Markdown format, and:
* It's an image link which needs to be formatted with embedded html grammar, please use global URL like ``https://user-images.githubusercontent.com/44491713/51381727-e3d0f780-1b4f-11e9-96ab-d26b9198ba65.png``\ , which can be automatically generated by dragging picture onto `Github Issue <https://github.com/Microsoft/nni/issues/new>`__ Box.
* It cannot be re-formatted by sphinx, such as source code, please use its global URL. For source code that links to our github repo, please use URLs rooted at ``https://github.com/Microsoft/nni/tree/v1.9/`` (\ :githublink:`mnist.py <examples/trials/mnist-tfv1/mnist.py>` for example).
%%%%%%
Code Styles & Naming Conventions
--------------------------------
* We follow `PEP8 <https://www.python.org/dev/peps/pep-0008/>`__ for Python code and naming conventions, do try to adhere to the same when making a pull request or making a change. One can also take the help of linters such as ``flake8`` or ``pylint``
* We also follow `NumPy Docstring Style <https://www.sphinx-doc.org/en/master/usage/extensions/example_numpy.html#example-numpy>`__ for Python Docstring Conventions. During the `documentation building <Contributing.rst#documentation>`__\ , we use `sphinx.ext.napoleon <https://www.sphinx-doc.org/en/master/usage/extensions/napoleon.html>`__ to generate Python API documentation from Docstring.
* For docstrings, please refer to `numpydoc docstring guide <https://numpydoc.readthedocs.io/en/latest/format.html>`__ and `pandas docstring guide <https://python-sprints.github.io/pandas/guide/pandas_docstring.html>`__
* For function docstring, **description**, **Parameters**, and **Returns** **Yields** are mandatory.
* For class docstring, **description**, **Attributes** are mandatory.
* For docstring to describe ``dict``, which is commonly used in our hyper-param format description, please refer to RiboKit Doc Standards
* `Internal Guideline on Writing Standards <https://ribokit.github.io/docs/text/>`__
Documentation
-------------
Our documentation is built with :githublink:`sphinx <docs>`.
* Before submitting the documentation change, please **build homepage locally**: ``cd docs/en_US && make html``, then you can see all the built documentation webpage under the folder ``docs/en_US/_build/html``. It's also highly recommended taking care of **every WARNING** during the build, which is very likely the signal of a **deadlink** and other annoying issues.
*
For links, please consider using **relative paths** first. However, if the documentation is written in Markdown format, and:
* It's an image link which needs to be formatted with embedded html grammar, please use global URL like ``https://user-images.githubusercontent.com/44491713/51381727-e3d0f780-1b4f-11e9-96ab-d26b9198ba65.png``, which can be automatically generated by dragging picture onto `Github Issue <https://github.com/Microsoft/nni/issues/new>`__ Box.
* It cannot be re-formatted by sphinx, such as source code, please use its global URL. For source code that links to our github repo, please use URLs rooted at ``https://github.com/Microsoft/nni/tree/v1.9/`` (:githublink:`mnist.py <examples/trials/mnist-tfv1/mnist.py>` for example).
* -
- Recommended
- Minimum
* - **Operating System**
- Ubuntu 16.04 or above
* - **CPU**
- Intel® Core™ i5 or AMD Phenom™ II X3 or better
- Intel® Core™ i3 or AMD Phenom™ X3 8650
* - **GPU**
- NVIDIA® GeForce® GTX 660 or better
- NVIDIA® GeForce® GTX 460
* - **Memory**
- 6 GB RAM
- 4 GB RAM
* - **Storage**
- 30 GB available hare drive space
* - **Internet**
- Boardband internet connection
* - **Resolution**
- 1024 x 768 minimum display resolution
%%%%%%
* -
- Recommended
- Minimum
* - **Operating System**
- Ubuntu 16.04 or above
-
* - **CPU**
- Intel® Core™ i5 or AMD Phenom™ II X3 or better
- Intel® Core™ i3 or AMD Phenom™ X3 8650
* - **GPU**
- NVIDIA® GeForce® GTX 660 or better
- NVIDIA® GeForce® GTX 460
* - **Memory**
- 6 GB RAM
- 4 GB RAM
* - **Storage**
- 30 GB available hare drive space
-
* - **Internet**
- Boardband internet connection
-
* - **Resolution**
- 1024 x 768 minimum display resolution
-
..
1.1 Declare NNI API
Include `import nni` in your trial code to use NNI APIs.
1.2 Get predefined parameters
Use the following code snippet:
RECEIVED_PARAMS = nni.get_next_parameter()
to get hyper-parameters' values assigned by tuner. `RECEIVED_PARAMS` is an object, for example:
{"conv_size": 2, "hidden_size": 124, "learning_rate": 0.0307, "dropout_rate": 0.2029}
1.3 Report NNI results
Use the API:
`nni.report_intermediate_result(accuracy)`
to send `accuracy` to assessor.
Use the API:
`nni.report_final_result(accuracy)`
to send `accuracy` to tuner.
%%%%%%
* Declare NNI API: include ``import nni`` in your trial code to use NNI APIs.
* Get predefined parameters
Use the following code snippet:
.. code-block:: python
RECEIVED_PARAMS = nni.get_next_parameter()
to get hyper-parameters' values assigned by tuner. ``RECEIVED_PARAMS`` is an object, for example:
.. code-block:: json
{"conv_size": 2, "hidden_size": 124, "learning_rate": 0.0307, "dropout_rate": 0.2029}
* Report NNI results: Use the API: ``nni.report_intermediate_result(accuracy)`` to send ``accuracy`` to assessor.
Use the API: ``nni.report_final_result(accuracy)`` to send `accuracy` to tuner.
* -
- Recommended
- Minimum
* - **Operating System**
- macOS 10.14.1 or above
* - **CPU**
- Intel® Core™ i7-4770 or better
- Intel® Core™ i5-760 or better
* - **GPU**
- AMD Radeon™ R9 M395X or better
- NVIDIA® GeForce® GT 750M or AMD Radeon™ R9 M290 or better
* - **Memory**
- 8 GB RAM
- 4 GB RAM
* - **Storage**
- 70GB available space SSD
- 70GB available space 7200 RPM HDD
* - **Internet**
- Boardband internet connection
* - **Resolution**
- 1024 x 768 minimum display resolution
%%%%%%
* -
- Recommended
- Minimum
* - **Operating System**
- macOS 10.14.1 or above
-
* - **CPU**
- Intel® Core™ i7-4770 or better
- Intel® Core™ i5-760 or better
* - **GPU**
- AMD Radeon™ R9 M395X or better
- NVIDIA® GeForce® GT 750M or AMD Radeon™ R9 M290 or better
* - **Memory**
- 8 GB RAM
- 4 GB RAM
* - **Storage**
- 70GB available space SSD
- 70GB available space 7200 RPM HDD
* - **Internet**
- Boardband internet connection
-
* - **Resolution**
- 1024 x 768 minimum display resolution
-
* -
- Recommended
- Minimum
* - **Operating System**
- Windows 10 1809 or above
* - **CPU**
- Intel® Core™ i5 or AMD Phenom™ II X3 or better
- Intel® Core™ i3 or AMD Phenom™ X3 8650
* - **GPU**
- NVIDIA® GeForce® GTX 660 or better
- NVIDIA® GeForce® GTX 460
* - **Memory**
- 6 GB RAM
- 4 GB RAM
* - **Storage**
- 30 GB available hare drive space
* - **Internet**
- Boardband internet connection
* - **Resolution**
- 1024 x 768 minimum display resolution
%%%%%%
* -
- Recommended
- Minimum
* - **Operating System**
- Windows 10 1809 or above
-
* - **CPU**
- Intel® Core™ i5 or AMD Phenom™ II X3 or better
- Intel® Core™ i3 or AMD Phenom™ X3 8650
* - **GPU**
- NVIDIA® GeForce® GTX 660 or better
- NVIDIA® GeForce® GTX 460
* - **Memory**
- 6 GB RAM
- 4 GB RAM
* - **Storage**
- 30 GB available hare drive space
-
* - **Internet**
- Boardband internet connection
-
* - **Resolution**
- 1024 x 768 minimum display resolution
-
* -
- s=4
- s=3
- s=2
- s=1
- s=0
* - i
- n r
- n r
- n r
- n r
- n r
* - 0
- 81 1
- 27 3
- 9 9
- 6 27
- 5 81
* - 1
- 27 3
- 9 9
- 3 27
- 2 81
-
* - 2
- 9 9
- 3 27
- 1 81
-
-
* - 3
- 3 27
- 1 81
-
-
-
* - 4
- 1 81
-
-
-
%%%%%%
* -
- s=4
- s=3
- s=2
- s=1
- s=0
* - i
- n r
- n r
- n r
- n r
- n r
* - 0
- 81 1
- 27 3
- 9 9
- 6 27
- 5 81
* - 1
- 27 3
- 9 9
- 3 27
- 2 81
-
* - 2
- 9 9
- 3 27
- 1 81
-
-
* - 3
- 3 27
- 1 81
-
-
-
* - 4
- 1 81
-
-
-
-
*Please refer to `here <https://nni.readthedocs.io/en/latest/sdk_reference.html>`__ for more APIs (e.g., ``nni.get_sequence_id()``\ ) provided by NNI.
%%%%%%
*Please refer to `here <https://nni.readthedocs.io/en/latest/sdk_reference.html>`__ for more APIs (e.g., ``nni.get_sequence_id()``\ ) provided by NNI.*
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment