[v2.0] Refactor code hierarchy (part 2) (#2987)

test/config/pr_tests.yml

@@ -47,8 +47,8 @@ testCases:
   config:
     maxTrialNum: 4
     trialConcurrency: 4
-  launchCommand: python3 -c 'from nni.nnicli import Experiment; exp = Experiment(); exp.start_experiment("$configFile")'
-  stopCommand: python3 -c 'from nni.nnicli import Experiment; exp = Experiment(); exp.connect_experiment("http://localhost:8080/"); exp.stop_experiment()'
+  launchCommand: python3 -c 'from nni.experiment import Experiment; exp = Experiment(); exp.start_experiment("$configFile")'
+  stopCommand: python3 -c 'from nni.experiment import Experiment; exp = Experiment(); exp.connect_experiment("http://localhost:8080/"); exp.stop_experiment()'
   validator:
     class: NnicliValidator
   platform: linux darwin

test/nni_test/nnitest/validators.py

@@ -6,8 +6,8 @@ from os import remove
 import subprocess
 import json
 import requests
-from nni.nnicli import Experiment
-from nni.nni_cmd.updater import load_search_space
+from nni.experiment import Experiment
+from nni.tools.nnictl.updater import load_search_space
 from utils import METRICS_URL, GET_IMPORTED_DATA_URL
 
 

ts/nni_manager/package.json

@@ -12,7 +12,6 @@
   "license": "MIT",
   "dependencies": {
     "azure-storage": "^2.10.2",
-    "chai-as-promised": "^7.1.1",
     "child-process-promise": "^2.2.1",
     "express": "^4.16.3",
     "express-joi-validator": "^2.0.0",
@@ -53,6 +52,7 @@
     "@typescript-eslint/eslint-plugin": "^2.10.0",
     "@typescript-eslint/parser": "^2.10.0",
     "chai": "^4.1.2",
+    "chai-as-promised": "^7.1.1",
     "eslint": "^6.7.2",
     "glob": "^7.1.3",
     "mocha": "^8.1.3",

ts/nni_manager/training_service/common/gpuData.ts

@@ -79,6 +79,6 @@ export function parseGpuIndices(gpuIndices?: string): Set<number> | undefined {
 export const GPU_INFO_COLLECTOR_FORMAT_WINDOWS: string =
     `
 $env:METRIC_OUTPUT_DIR="{0}"
-$app = Start-Process "python" -ArgumentList "-m nni_gpu_tool.gpu_metrics_collector" -passthru -NoNewWindow
+$app = Start-Process "python" -ArgumentList "-m nni.tools.gpu_tool.gpu_metrics_collector" -passthru -NoNewWindow
 Write $app.ID | Out-File {1} -NoNewline -encoding utf8
 `;

ts/nni_manager/training_service/common/util.ts

@@ -236,7 +236,7 @@ export function getScriptName(fileNamePrefix: string): string {
 }
 
 export function getGpuMetricsCollectorBashScriptContent(scriptFolder: string): string {
-    return `echo $$ > ${scriptFolder}/pid ; METRIC_OUTPUT_DIR=${scriptFolder} python3 -m nni_gpu_tool.gpu_metrics_collector`;
+    return `echo $$ > ${scriptFolder}/pid ; METRIC_OUTPUT_DIR=${scriptFolder} python3 -m nni.tools.gpu_tool.gpu_metrics_collector`;
 }
 
 export function runGpuMetricsCollector(scriptFolder: string): void {

ts/nni_manager/training_service/dlts/dltsData.ts

@@ -4,5 +4,5 @@
 export const DLTS_TRIAL_COMMAND_FORMAT: string =
 `export NNI_PLATFORM=dlts NNI_SYS_DIR={0} NNI_OUTPUT_DIR={1} NNI_TRIAL_JOB_ID={2} NNI_EXP_ID={3} NNI_TRIAL_SEQ_ID={4} MULTI_PHASE={5} \
 && cd $NNI_SYS_DIR && sh install_nni.sh \
-&& cd '{6}' && python3 -m nni_trial_tool.trial_keeper --trial_command '{7}' \
+&& cd '{6}' && python3 -m nni.tools.trial_tool.trial_keeper --trial_command '{7}' \
 --nnimanager_ip '{8}' --nnimanager_port '{9}' --nni_manager_version '{10}' --log_collection '{11}'`;

ts/nni_manager/training_service/kubernetes/kubernetesData.ts

@@ -52,5 +52,5 @@ mkdir -p $NNI_OUTPUT_DIR
 cp -r $NNI_CODE_DIR/. $NNI_SYS_DIR/code
 sh $NNI_SYS_DIR/install_nni.sh
 cd $NNI_SYS_DIR/code
-python3 -m nni_trial_tool.trial_keeper --trial_command '{8}' --nnimanager_ip {9} --nnimanager_port {10} \
+python3 -m nni.tools.trial_tool.trial_keeper --trial_command '{8}' --nnimanager_ip {9} --nnimanager_port {10} \
 --nni_manager_version '{11}' --log_collection '{12}' 1>$NNI_OUTPUT_DIR/trialkeeper_stdout 2>$NNI_OUTPUT_DIR/trialkeeper_stderr`;

ts/nni_manager/training_service/pai/paiK8S/paiK8SData.ts

@@ -16,5 +16,5 @@ fi`;
 export const PAI_K8S_TRIAL_COMMAND_FORMAT: string =
 `export NNI_PLATFORM=pai NNI_SYS_DIR={0} NNI_OUTPUT_DIR={1} NNI_TRIAL_JOB_ID={2} NNI_EXP_ID={3} NNI_TRIAL_SEQ_ID={4} MULTI_PHASE={5} \
 && NNI_CODE_DIR={6} && mkdir -p $NNI_SYS_DIR/code && cp -r $NNI_CODE_DIR/. $NNI_SYS_DIR/code && sh $NNI_SYS_DIR/install_nni.sh \
-&& cd $NNI_SYS_DIR/code && python3 -m nni_trial_tool.trial_keeper --trial_command '{7}' --nnimanager_ip '{8}' --nnimanager_port '{9}' \
+&& cd $NNI_SYS_DIR/code && python3 -m nni.tools.trial_tool.trial_keeper --trial_command '{7}' --nnimanager_ip '{8}' --nnimanager_port '{9}' \
 --nni_manager_version '{10}' --log_collection '{11}' | tee $NNI_OUTPUT_DIR/trial.log`;

ts/nni_manager/training_service/remote_machine/extends/linuxCommands.ts

@@ -26,7 +26,7 @@ class LinuxCommands extends OsCommands {
             cp -r $NNI_CODE_DIR/. $NNI_SYS_DIR/code
             sh $NNI_SYS_DIR/install_nni.sh
             cd $NNI_SYS_DIR/code
-            python3 -m nni_trial_tool.trial_keeper --trial_command '${cudaVisibleSetting} ${command}' --nnimanager_ip '${nniManagerAddress}' \
+            python3 -m nni.tools.trial_tool.trial_keeper --trial_command '${cudaVisibleSetting} ${command}' --nnimanager_ip '${nniManagerAddress}' \
                 --nnimanager_port '${nniManagerPort}' --nni_manager_version '${nniManagerVersion}' \
                 --job_id_file ${jobIdFileName} \
                 --log_collection '${logCollection}' 1>$NNI_OUTPUT_DIR/trialkeeper_stdout 2>$NNI_OUTPUT_DIR/trialkeeper_stderr
@@ -34,7 +34,7 @@ class LinuxCommands extends OsCommands {
     }
 
     public generateGpuStatsScript(scriptFolder: string): string {
-        return `echo $$ > ${scriptFolder}/pid ; METRIC_OUTPUT_DIR=${scriptFolder} python3 -m nni_gpu_tool.gpu_metrics_collector`;
+        return `echo $$ > ${scriptFolder}/pid ; METRIC_OUTPUT_DIR=${scriptFolder} python3 -m nni.tools.gpu_tool.gpu_metrics_collector`;
     }
 
     public createFolder(folderName: string, sharedFolder: boolean = false): string {

ts/nni_manager/training_service/remote_machine/extends/windowsCommands.ts

@@ -38,7 +38,7 @@ class WindowsCommands extends OsCommands {
             )
 
             echo starting script
-            python -m nni_trial_tool.trial_keeper --trial_command "${command}" --nnimanager_ip "${nniManagerAddress}" --nnimanager_port "${nniManagerPort}" --nni_manager_version "${nniManagerVersion}" --log_collection "${logCollection}" --job_id_file ${jobIdFileName} 1>%NNI_OUTPUT_DIR%/trialkeeper_stdout 2>%NNI_OUTPUT_DIR%/trialkeeper_stderr
+            python -m nni.tools.trial_tool.trial_keeper --trial_command "${command}" --nnimanager_ip "${nniManagerAddress}" --nnimanager_port "${nniManagerPort}" --nni_manager_version "${nniManagerVersion}" --log_collection "${logCollection}" --job_id_file ${jobIdFileName} 1>%NNI_OUTPUT_DIR%/trialkeeper_stdout 2>%NNI_OUTPUT_DIR%/trialkeeper_stderr
 
             echo save exit code(%ERRORLEVEL%) and time
             echo|set /p="%ERRORLEVEL% " > ${exitCodeFile}
@@ -46,7 +46,7 @@ class WindowsCommands extends OsCommands {
     }
 
     public generateGpuStatsScript(scriptFolder: string): string {
-        return `powershell -command $env:Path=If($env:prePath){$env:prePath}Else{$env:Path};$env:METRIC_OUTPUT_DIR='${scriptFolder}';$app = Start-Process -FilePath python -NoNewWindow -passthru -ArgumentList '-m nni_gpu_tool.gpu_metrics_collector' -RedirectStandardOutput ${scriptFolder}\\scriptstdout -RedirectStandardError ${scriptFolder}\\scriptstderr;Write $PID ^| Out-File ${scriptFolder}\\pid -NoNewline -encoding utf8;wait-process $app.ID`;
+        return `powershell -command $env:Path=If($env:prePath){$env:prePath}Else{$env:Path};$env:METRIC_OUTPUT_DIR='${scriptFolder}';$app = Start-Process -FilePath python -NoNewWindow -passthru -ArgumentList '-m nni.tools.gpu_tool.gpu_metrics_collector' -RedirectStandardOutput ${scriptFolder}\\scriptstdout -RedirectStandardError ${scriptFolder}\\scriptstderr;Write $PID ^| Out-File ${scriptFolder}\\pid -NoNewline -encoding utf8;wait-process $app.ID`;
     }
 
     public createFolder(folderName: string, sharedFolder: boolean = false): string {

ts/nni_manager/training_service/reusable/trialDispatcher.ts

@@ -228,6 +228,7 @@ class TrialDispatcher implements TrainingService {
         const runnerSettings = storageService.joinPath(envDir, "settings.json");
         await storageService.save(JSON.stringify(this.runnerSettings), runnerSettings);
 
+        // FIXME: what the hell is this?
         if (this.isDeveloping) {
             let trialToolsPath = path.join(__dirname, "../../../../../tools/nni_trial_tool");
             if (false === fs.existsSync(trialToolsPath)) {
@@ -598,7 +599,7 @@ class TrialDispatcher implements TrainingService {
         const envName = `nni_exp_${this.experimentId}_env_${envId}`;
         const environment = environmentService.createEnvironmentInformation(envId, envName);
 
-        environment.command = `sh ../install_nni.sh && python3 -m nni_trial_tool.trial_runner`;
+        environment.command = `sh ../install_nni.sh && python3 -m nni.tools.trial_tool.trial_runner`;
 
         if (this.isDeveloping) {
             environment.command = "[ -d \"nni_trial_tool\" ] && echo \"nni_trial_tool exists already\" || (mkdir ./nni_trial_tool && tar -xof ../nni_trial_tool.tar.gz -C ./nni_trial_tool) && pip3 install websockets && " + environment.command;