[v2.0] Refactor code hierarchy (part 2) (#2987)

.gitignore

-/nni_node/node
-/nni_node/node.exe
-/toolchain
+/nni_node/
+/toolchain/
+
+# unit test generated files
+/test/model_path/
+/test/temp.json
+/test/ut/sdk/*.pth
 
 
 # Logs

Makefile

-SHELL := /bin/bash
-
-_INFO := $(shell echo -e '\033[1;36m')
-_END := $(shell echo -e '\033[0m')
-
-## Detect OS
-UNAME_S := $(shell uname -s)
-ifeq ($(UNAME_S), Linux)
-    OS_SPEC := linux
-else ifeq ($(UNAME_S), Darwin)
-    OS_SPEC := darwin
-else
-    $(error platform $(UNAME_S) not supported)
-endif
-
-_PWD := $(PWD)
-YARN ?= PATH=$(_PWD)/nni_node:$${PATH} $(PWD)/toolchain/yarn/bin/yarn
-
-
-.PHONY: build
-build: nni_node/node create-symlinks
-	#$(_INFO) Building NNI Manager $(_END)
-	cd ts/nni_manager && $(YARN) && $(YARN) build
-	cp -rf ts/nni_manager/config ts/nni_manager/dist/
-	#$(_INFO) Building WebUI $(_END)
-	cd ts/webui && $(YARN) && $(YARN) build
-	#$(_INFO) Building NAS UI $(_END)
-	cd ts/nasui && $(YARN) && $(YARN) build
-
-
-nni_node/node:
-	mkdir -p toolchain
-	wget https://nodejs.org/dist/v10.22.1/node-v10.22.1-$(OS_SPEC)-x64.tar.xz -O toolchain/node.tar.xz
-	wget https://github.com/yarnpkg/yarn/releases/download/v1.22.10/yarn-v1.22.10.tar.gz -O toolchain/yarn.tar.gz
-	
-	mkdir -p toolchain/node toolchain/yarn
-	tar -xf toolchain/node.tar.xz -C toolchain/node --strip-components 1
-	tar -xf toolchain/yarn.tar.gz -C toolchain/yarn --strip-components 1
-	
-	cp toolchain/node/bin/node nni_node/
-
-
-.PHONY: create-symlinks
-create-symlinks:
-	ln -sfT ../ts/nni_manager/dist/common nni_node/common
-	ln -sfT ../ts/nni_manager/dist/config nni_node/config
-	ln -sfT ../ts/nni_manager/dist/core nni_node/core
-	ln -sfT ../ts/nni_manager/dist/rest_server nni_node/rest_server
-	ln -sfT ../ts/nni_manager/dist/training_service nni_node/training_service
-	ln -sfT ../ts/nni_manager/dist/main.js nni_node/main.js
-	ln -sfT ../ts/nni_manager/package.json nni_node/package.json
-	ln -sfT ../ts/nni_manager/node_modules nni_node/node_modules
-	
-	ln -sfT ../ts/webui/build nni_node/static
-	
-	mkdir -p nni_node/nasui
-	ln -sfT ../../ts/nasui/build nni_node/nasui/build
-	ln -sfT ../../ts/nasui/server.js nni_node/nasui/server.js
-
-
-.PHONY: clean
-clean:
-	-rm nni_node/node
-	-rm -rf toolchain
-	-rm -rf nni_node/common
-	-rm -rf nni_node/config
-	-rm -rf nni_node/core
-	-rm -rf nni_node/rest_server
-	-rm -rf nni_node/training_service
-	-rm -rf nni_node/main.js
-	-rm -rf nni_node/package.json
-	-rm -rf nni_node/node_modules
-	-rm -rf nni_node/nasui
-	-rm -rf nni_node/static

azure-pipelines.yml

@@ -14,8 +14,7 @@ jobs:
       echo "##vso[task.setvariable variable=PATH]${HOME}/.local/bin:${PATH}"
     displayName: 'Install python tools'
   - script: |
-      make
-      python3 -m pip install -U -e .
+      python3 setup.py develop
     displayName: 'Install nni toolkit via source code'
   - script: |
       set -e
@@ -46,10 +45,10 @@ jobs:
       cd test
       python3 nni_test/nnitest/run_tests.py --config config/pr_tests.yml
     displayName: 'Simple test'
-  - script: |
-      cd docs/en_US/
-      sphinx-build -M html . _build -W
-    displayName: 'Sphinx Documentation Build check'
+  #- script: |
+  #    cd docs/en_US/
+  #    sphinx-build -M html . _build -W
+  #  displayName: 'Sphinx Documentation Build check'
 
 - job: 'ubuntu_1804_python36_legacy_torch_tf'
   pool:
@@ -65,8 +64,7 @@ jobs:
       echo "##vso[task.setvariable variable=PATH]${HOME}/.local/bin:${PATH}"
     displayName: 'Install python tools'
   - script: |
-      make
-      python3 -m pip install -U -e .
+      python3 setup.py develop
     displayName: 'Install nni toolkit via source code'
   - script: |
       set -e

docs/en_US/nnicli_ref.md

@@ -3,7 +3,7 @@
 NNI client is a python API of `nnictl`, which implements the most commonly used commands. Users can use this API to control their experiments, collect experiment results and conduct advanced analyses based on experiment results in python code directly instead of using command line. Here is an example:
 
 ```
-from nni.nnicli import Experiment
+from nni.experiment import Experiment
 
 # create an experiment instance
 exp = Experiment() 
@@ -28,14 +28,14 @@ exp.stop_experiment()
 ## References
 
 ```eval_rst
-..  autoclass:: nni.nnicli.Experiment
+..  autoclass:: nni.experiment.Experiment
     :members:
-..  autoclass:: nni.nnicli.TrialJob
+..  autoclass:: nni.experiment.TrialJob
     :members:
-..  autoclass:: nni.nnicli.TrialHyperParameters
+..  autoclass:: nni.experiment.TrialHyperParameters
     :members:
-..  autoclass:: nni.nnicli.TrialMetricData
+..  autoclass:: nni.experiment.TrialMetricData
     :members:
-..  autoclass:: nni.nnicli.TrialResult
+..  autoclass:: nni.experiment.TrialResult
     :members:
 ```

nni/__init__.py

@@ -3,13 +3,13 @@
 
 __version__ = '999.0.0-developing'
 
-from .env_vars import dispatcher_env_vars
+from .runtime.env_vars import dispatcher_env_vars
 from .utils import ClassArgsValidator
 
 if dispatcher_env_vars.SDK_PROCESS != 'dispatcher':
     from .trial import *
     from .smartparam import *
-    from .nas_utils import training_update
+    from .common.nas_utils import training_update
 
 class NoMoreTrialError(Exception):
     def __init__(self, ErrorInfo):

nni/__main__.py

@@ -7,9 +7,9 @@ import logging
 import json
 import base64
 
-from .common import enable_multi_thread, enable_multi_phase
-from .msg_dispatcher import MsgDispatcher
-from .package_utils import create_builtin_class_instance, create_customized_class_instance
+from .runtime.common import enable_multi_thread, enable_multi_phase
+from .runtime.msg_dispatcher import MsgDispatcher
+from .tools.package_utils import create_builtin_class_instance, create_customized_class_instance
 
 logger = logging.getLogger('nni.main')
 logger.debug('START')

nni/compression/torch/pruning/admm_pruner.py → nni/algorithms/compression/pytorch/pruning/admm_pruner.py

@@ -5,7 +5,7 @@ import logging
 import torch
 from schema import And, Optional
 
-from ..utils.config_validation import CompressorSchema
+from nni.compression.pytorch.utils.config_validation import CompressorSchema
 from .constants import MASKER_DICT
 from .one_shot import OneshotPruner
 

nni/compression/torch/pruning/agp.py → nni/algorithms/compression/pytorch/pruning/agp.py

@@ -13,8 +13,8 @@ import logging
 import torch
 from schema import And, Optional
 from .constants import MASKER_DICT
-from ..utils.config_validation import CompressorSchema
-from ..compressor import Pruner
+from nni.compression.pytorch.utils.config_validation import CompressorSchema
+from nni.compression.pytorch.compressor import Pruner
 
 __all__ = ['AGPPruner']
 

nni/compression/torch/pruning/amc/amc_pruner.py → nni/algorithms/compression/pytorch/pruning/amc/amc_pruner.py

@@ -9,7 +9,7 @@ import numpy as np
 import torch
 from torch.utils.tensorboard import SummaryWriter
 
-from nni.compression.torch.compressor import Pruner
+from nni.compression.pytorch.compressor import Pruner
 from .channel_pruning_env import ChannelPruningEnv
 from .lib.agent import DDPG
 from .lib.utils import get_output_folder

nni/compression/torch/pruning/amc/channel_pruning_env.py → nni/algorithms/compression/pytorch/pruning/amc/channel_pruning_env.py

@@ -10,7 +10,7 @@ import numpy as np
 import torch
 import torch.nn as nn
 
-from nni.compression.torch.compressor import PrunerModuleWrapper
+from nni.compression.pytorch.compressor import PrunerModuleWrapper
 from .. import AMCWeightMasker
 
 _logger = logging.getLogger(__name__)

nni/compression/torch/pruning/auto_compress_pruner.py → nni/algorithms/compression/pytorch/pruning/auto_compress_pruner.py

@@ -8,10 +8,10 @@ import torch
 from schema import And, Optional
 
 from nni.utils import OptimizeMode
-from nni.compression.torch import ModelSpeedup
+from nni.compression.pytorch import ModelSpeedup
 
-from ..compressor import Pruner
-from ..utils.config_validation import CompressorSchema
+from nni.compression.pytorch.compressor import Pruner
+from nni.compression.pytorch.utils.config_validation import CompressorSchema
 from .simulated_annealing_pruner import SimulatedAnnealingPruner
 from .admm_pruner import ADMMPruner
 

nni/compression/torch/pruning/constants.py → nni/algorithms/compression/pytorch/pruning/constants.py

@@ -2,7 +2,7 @@
 # Licensed under the MIT license.
 
 
-from ..pruning import LevelPrunerMasker, SlimPrunerMasker, L1FilterPrunerMasker, \
+from . import LevelPrunerMasker, SlimPrunerMasker, L1FilterPrunerMasker, \
     L2FilterPrunerMasker, FPGMPrunerMasker, TaylorFOWeightFilterPrunerMasker, \
     ActivationAPoZRankFilterPrunerMasker, ActivationMeanRankFilterPrunerMasker