Merge pull request #3030 from microsoft/v2.0

Merge v2.0 into master

azure-pipelines.yml

@@ -14,11 +14,11 @@ jobs:
       echo "##vso[task.setvariable variable=PATH]${HOME}/.local/bin:${PATH}"
     displayName: 'Install python tools'
   - script: |
-      source install.sh
+      python3 setup.py develop
     displayName: 'Install nni toolkit via source code'
   - script: |
       set -e
-      cd src/nni_manager
+      cd ts/nni_manager
       yarn eslint
       # uncomment following 2 lines to enable webui eslint
       cd ../webui
@@ -45,10 +45,10 @@ jobs:
       cd test
       python3 nni_test/nnitest/run_tests.py --config config/pr_tests.yml
     displayName: 'Simple test'
-  - script: |
-      cd docs/en_US/
-      sphinx-build -M html . _build -W
-    displayName: 'Sphinx Documentation Build check'
+  #- script: |
+  #    cd docs/en_US/
+  #    sphinx-build -M html . _build -W
+  #  displayName: 'Sphinx Documentation Build check'
 
 - job: 'ubuntu_1804_python36_legacy_torch_tf'
   pool:
@@ -64,7 +64,7 @@ jobs:
       echo "##vso[task.setvariable variable=PATH]${HOME}/.local/bin:${PATH}"
     displayName: 'Install python tools'
   - script: |
-      source install.sh
+      python3 setup.py develop
     displayName: 'Install nni toolkit via source code'
   - script: |
       set -e
@@ -78,18 +78,14 @@ jobs:
     displayName: 'Install dependencies'
   - script: |
       set -e
-      python3 -m pylint --rcfile pylintrc nni_annotation
-      python3 -m pylint --rcfile pylintrc nni_cmd
-      python3 -m pylint --rcfile pylintrc nni_gpu_tool
-      python3 -m pylint --rcfile pylintrc nni_trial_tool
       python3 -m pylint --rcfile pylintrc nni
-      python3 -m pylint --rcfile pylintrc nnicli
     displayName: 'Run pylint'
   - script: |
       set -e
       python3 -m pip install flake8 --user
-      EXCLUDES=./src/nni_manager/,./src/webui,./tools/nni_annotation/testcase/,./examples/trials/mnist-nas/*/mnist*.py,./examples/trials/nas_cifar10/src/cifar10/general_child.py
-      python3 -m flake8 . --count --exclude=$EXCLUDES --select=E9,F63,F72,F82 --show-source --statistics
+      python3 -m flake8 nni --count --select=E9,F63,F72,F82 --show-source --statistics
+      EXCLUDES=examples/trials/mnist-nas/*/mnist*.py,examples/trials/nas_cifar10/src/cifar10/general_child.py
+      python3 -m flake8 examples --count --exclude=$EXCLUDES --select=E9,F63,F72,F82 --show-source --statistics
     displayName: 'Run flake8 tests to find Python syntax errors and undefined names'
   - script: |
       cd test
@@ -101,58 +97,58 @@ jobs:
     displayName: 'Simple test'
 
 
-- job: 'macos_latest_python38'
-  pool:
-    vmImage: 'macOS-latest'
-
-  steps:
-  - script: |
-      export PYTHON38_BIN_DIR=/usr/local/Cellar/python@3.8/`ls /usr/local/Cellar/python@3.8`/bin
-      echo "##vso[task.setvariable variable=PATH]${PYTHON38_BIN_DIR}:${HOME}/Library/Python/3.8/bin:${PATH}"
-      python3 -m pip install --upgrade pip setuptools
-    displayName: 'Install python tools'
-  - script: |
-      echo "network-timeout 600000" >> ${HOME}/.yarnrc
-      source install.sh
-    displayName: 'Install nni toolkit via source code'
-  - script: |
-      set -e
-      # pytorch Mac binary does not support CUDA, default is cpu version
-      python3 -m pip install torchvision==0.6.0 torch==1.5.0 --user
-      python3 -m pip install tensorflow==2.2 --user
-      brew install swig@3
-      rm -f /usr/local/bin/swig
-      ln -s /usr/local/opt/swig\@3/bin/swig /usr/local/bin/swig
-      nnictl package install --name=SMAC
-    displayName: 'Install dependencies'
-  - script: |
-      cd test
-      source scripts/unittest.sh
-    displayName: 'Unit test'
-  - script: |
-      cd test
-      python3 nni_test/nnitest/run_tests.py --config config/pr_tests.yml
-    displayName: 'Simple test'
-
-- job: 'win2016_python37'
-  pool:
-    vmImage: 'vs2017-win2016'
-
-  steps:
-  - script: |
-      powershell.exe -file install.ps1
-    displayName: 'Install nni toolkit via source code'
-  - script: |
-      python -m pip install scikit-learn==0.23.2 --user
-      python -m pip install keras==2.1.6 --user
-      python -m pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html --user
-      python -m pip install tensorflow==1.15.2 --user
-    displayName: 'Install dependencies'
-  - script: |
-      cd test
-      powershell.exe -file scripts/unittest.ps1
-    displayName: 'unit test'
-  - script: |
-      cd test
-      python nni_test/nnitest/run_tests.py --config config/pr_tests.yml
-    displayName: 'Simple test'
+#- job: 'macos_latest_python38'
+#  pool:
+#    vmImage: 'macOS-latest'
+#
+#  steps:
+#  - script: |
+#      export PYTHON38_BIN_DIR=/usr/local/Cellar/python@3.8/`ls /usr/local/Cellar/python@3.8`/bin
+#      echo "##vso[task.setvariable variable=PATH]${PYTHON38_BIN_DIR}:${HOME}/Library/Python/3.8/bin:${PATH}"
+#      python3 -m pip install --upgrade pip setuptools
+#    displayName: 'Install python tools'
+#  - script: |
+#      echo "network-timeout 600000" >> ${HOME}/.yarnrc
+#      source install.sh
+#    displayName: 'Install nni toolkit via source code'
+#  - script: |
+#      set -e
+#      # pytorch Mac binary does not support CUDA, default is cpu version
+#      python3 -m pip install torchvision==0.6.0 torch==1.5.0 --user
+#      python3 -m pip install tensorflow==2.2 --user
+#      brew install swig@3
+#      rm -f /usr/local/bin/swig
+#      ln -s /usr/local/opt/swig\@3/bin/swig /usr/local/bin/swig
+#      nnictl package install --name=SMAC
+#    displayName: 'Install dependencies'
+#  - script: |
+#      cd test
+#      source scripts/unittest.sh
+#    displayName: 'Unit test'
+#  - script: |
+#      cd test
+#      python3 nni_test/nnitest/run_tests.py --config config/pr_tests.yml
+#    displayName: 'Simple test'
+#
+#- job: 'win2016_python37'
+#  pool:
+#    vmImage: 'vs2017-win2016'
+#
+#  steps:
+#  - script: |
+#      powershell.exe -file install.ps1
+#    displayName: 'Install nni toolkit via source code'
+#  - script: |
+#      python -m pip install scikit-learn==0.23.2 --user
+#      python -m pip install keras==2.1.6 --user
+#      python -m pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html --user
+#      python -m pip install tensorflow==1.15.2 --user
+#    displayName: 'Install dependencies'
+#  - script: |
+#      cd test
+#      powershell.exe -file scripts/unittest.ps1
+#    displayName: 'unit test'
+#  - script: |
+#      cd test
+#      python nni_test/nnitest/run_tests.py --config config/pr_tests.yml
+#    displayName: 'Simple test'

docs/en_US/nnicli_ref.md

@@ -3,7 +3,7 @@
 NNI client is a python API of `nnictl`, which implements the most commonly used commands. Users can use this API to control their experiments, collect experiment results and conduct advanced analyses based on experiment results in python code directly instead of using command line. Here is an example:
 
 ```
-from nnicli import Experiment
+from nni.experiment import Experiment
 
 # create an experiment instance
 exp = Experiment() 
@@ -28,14 +28,14 @@ exp.stop_experiment()
 ## References
 
 ```eval_rst
-..  autoclass:: nnicli.Experiment
+..  autoclass:: nni.experiment.Experiment
     :members:
-..  autoclass:: nnicli.TrialJob
+..  autoclass:: nni.experiment.TrialJob
     :members:
-..  autoclass:: nnicli.TrialHyperParameters
+..  autoclass:: nni.experiment.TrialHyperParameters
     :members:
-..  autoclass:: nnicli.TrialMetricData
+..  autoclass:: nni.experiment.TrialMetricData
     :members:
-..  autoclass:: nnicli.TrialResult
+..  autoclass:: nni.experiment.TrialResult
     :members:
 ```

src/sdk/pynni/nni/__init__.py → nni/__init__.py

@@ -3,13 +3,13 @@
 
 __version__ = '999.0.0-developing'
 
-from .env_vars import dispatcher_env_vars
+from .runtime.env_vars import dispatcher_env_vars
 from .utils import ClassArgsValidator
 
 if dispatcher_env_vars.SDK_PROCESS != 'dispatcher':
     from .trial import *
     from .smartparam import *
-    from .nas_utils import training_update
+    from .common.nas_utils import training_update
 
 class NoMoreTrialError(Exception):
     def __init__(self, ErrorInfo):

src/sdk/pynni/nni/__main__.py → nni/__main__.py

@@ -7,9 +7,9 @@ import logging
 import json
 import base64
 
-from .common import enable_multi_thread, enable_multi_phase
-from .msg_dispatcher import MsgDispatcher
-from .package_utils import create_builtin_class_instance, create_customized_class_instance
+from .runtime.common import enable_multi_thread, enable_multi_phase
+from .runtime.msg_dispatcher import MsgDispatcher
+from .tools.package_utils import create_builtin_class_instance, create_customized_class_instance
 
 logger = logging.getLogger('nni.main')
 logger.debug('START')

src/sdk/pynni/nni/compression/torch/pruning/admm_pruner.py → nni/algorithms/compression/pytorch/pruning/admm_pruner.py

@@ -5,7 +5,7 @@ import logging
 import torch
 from schema import And, Optional
 
-from ..utils.config_validation import CompressorSchema
+from nni.compression.pytorch.utils.config_validation import CompressorSchema
 from .constants import MASKER_DICT
 from .one_shot import OneshotPruner
 

src/sdk/pynni/nni/compression/torch/pruning/agp.py → nni/algorithms/compression/pytorch/pruning/agp.py

@@ -13,8 +13,8 @@ import logging
 import torch
 from schema import And, Optional
 from .constants import MASKER_DICT
-from ..utils.config_validation import CompressorSchema
-from ..compressor import Pruner
+from nni.compression.pytorch.utils.config_validation import CompressorSchema
+from nni.compression.pytorch.compressor import Pruner
 
 __all__ = ['AGPPruner']
 

src/sdk/pynni/nni/compression/torch/pruning/amc/amc_pruner.py → nni/algorithms/compression/pytorch/pruning/amc/amc_pruner.py

@@ -9,7 +9,7 @@ import numpy as np
 import torch
 from torch.utils.tensorboard import SummaryWriter
 
-from nni.compression.torch.compressor import Pruner
+from nni.compression.pytorch.compressor import Pruner
 from .channel_pruning_env import ChannelPruningEnv
 from .lib.agent import DDPG
 from .lib.utils import get_output_folder

src/sdk/pynni/nni/compression/torch/pruning/amc/channel_pruning_env.py → nni/algorithms/compression/pytorch/pruning/amc/channel_pruning_env.py

@@ -10,7 +10,7 @@ import numpy as np
 import torch
 import torch.nn as nn
 
-from nni.compression.torch.compressor import PrunerModuleWrapper
+from nni.compression.pytorch.compressor import PrunerModuleWrapper
 from .. import AMCWeightMasker
 
 _logger = logging.getLogger(__name__)

src/sdk/pynni/nni/compression/torch/pruning/auto_compress_pruner.py → nni/algorithms/compression/pytorch/pruning/auto_compress_pruner.py

@@ -8,10 +8,10 @@ import torch
 from schema import And, Optional
 
 from nni.utils import OptimizeMode
-from nni.compression.torch import ModelSpeedup
+from nni.compression.pytorch import ModelSpeedup
 
-from ..compressor import Pruner
-from ..utils.config_validation import CompressorSchema
+from nni.compression.pytorch.compressor import Pruner
+from nni.compression.pytorch.utils.config_validation import CompressorSchema
 from .simulated_annealing_pruner import SimulatedAnnealingPruner
 from .admm_pruner import ADMMPruner
 

src/sdk/pynni/nni/compression/torch/pruning/constants.py → nni/algorithms/compression/pytorch/pruning/constants.py

@@ -2,7 +2,7 @@
 # Licensed under the MIT license.
 
 
-from ..pruning import LevelPrunerMasker, SlimPrunerMasker, L1FilterPrunerMasker, \
+from . import LevelPrunerMasker, SlimPrunerMasker, L1FilterPrunerMasker, \
     L2FilterPrunerMasker, FPGMPrunerMasker, TaylorFOWeightFilterPrunerMasker, \
     ActivationAPoZRankFilterPrunerMasker, ActivationMeanRankFilterPrunerMasker