Support nas installation (#2217)

* check pylint for nni_cmd

* fix id error

* init

* remove comments
Co-authored-by: SparkSnail <Administrator@MININT-27KBA7M.fareast.corp.microsoft.com>

Makefile

@@ -45,6 +45,7 @@ NNI_INSTALL_PATH ?= $(INSTALL_PREFIX)/nni
 
 BIN_FOLDER ?= $(ROOT_FOLDER)/bin
 NNI_PKG_FOLDER ?= $(ROOT_FOLDER)/nni
+NASUI_PKG_FOLDER ?= $(ROOT_FOLDER)/nni/nasui
 
 ## Dependency information
 NNI_DEPENDENCY_FOLDER = /tmp/$(USER)
@@ -132,6 +133,8 @@ clean:
 	-rm -rf src/sdk/pynni/nni_sdk.egg-info
 	-rm -rf src/webui/build
 	-rm -rf src/webui/node_modules
+	-rm -rf src/nasui/build
+	-rm -rf src/nasui/node_modules
 
 # Main targets end
 
@@ -193,6 +196,11 @@ install-node-modules:
 	sed -ie 's/$(NNI_VERSION_TEMPLATE)/$(NNI_VERSION_VALUE)/' $(NNI_PKG_FOLDER)/package.json
 	$(NNI_YARN) --prod --cwd $(NNI_PKG_FOLDER)
 	cp -r src/webui/build $(NNI_PKG_FOLDER)/static
+	# Install nasui
+	mkdir -p $(NASUI_PKG_FOLDER)
+	cp -rf src/nasui/build $(NASUI_PKG_FOLDER)
+	cp src/nasui/server.js $(NASUI_PKG_FOLDER)
+
 
 .PHONY: dev-install-node-modules
 dev-install-node-modules:
@@ -203,6 +211,8 @@ dev-install-node-modules:
 	sed -ie 's/$(NNI_VERSION_TEMPLATE)/$(NNI_VERSION_VALUE)/' $(NNI_PKG_FOLDER)/package.json
 	ln -sf ${PWD}/src/nni_manager/node_modules $(NNI_PKG_FOLDER)/node_modules
 	ln -sf ${PWD}/src/webui/build $(NNI_PKG_FOLDER)/static
+	ln -sf ${PWD}/src/nasui/build $(NASUI_PKG_FOLDER)/build
+	ln -sf ${PWD}/src/nasui/server.js $(NASUI_PKG_FOLDER)/server.js
 
 .PHONY: install-scripts
 install-scripts:

deployment/pypi/Makefile

@@ -36,10 +36,13 @@ build:
 	tar -xf $(NNI_YARN_TARBALL) -C $(NNI_YARN_FOLDER) --strip-components 1
 	cd $(CWD)../../src/nni_manager && $(NNI_YARN) && $(NNI_YARN) build
 	cd $(CWD)../../src/webui && $(NNI_YARN) && $(NNI_YARN) build
+	cd $(CWD)../../src/nasui && $(NNI_YARN) && $(NNI_YARN) build
 	rm -rf $(CWD)nni
 	cp -r $(CWD)../../src/nni_manager/dist $(CWD)nni
 	cp -r $(CWD)../../src/nni_manager/config $(CWD)nni
 	cp -r $(CWD)../../src/webui/build $(CWD)nni/static
+	cp -r $(CWD)../../src/nasui/build $(CWD)nni/nasui
+	cp $(CWD)../../src/nasui/server.js $(CWD)nni/nasui
 	cp $(CWD)../../src/nni_manager/package.json $(CWD)nni
 	sed -ie 's/$(NNI_VERSION_TEMPLATE)/$(NNI_VERSION_VALUE)/' $(CWD)nni/package.json
 	cd $(CWD)nni && $(NNI_YARN) --prod

deployment/pypi/install.ps1

@@ -46,11 +46,16 @@ yarn build
 cd $CWD\..\..\src\webui
 yarn
 yarn build
+cd $CWD\..\..\src\nasui
+yarn
+yarn build
 if(Test-Path $CWD\nni){
     Remove-Item $CWD\nni -r -fo
 }
 Copy-Item $CWD\..\..\src\nni_manager\dist $CWD\nni -Recurse
 Copy-Item $CWD\..\..\src\webui\build $CWD\nni\static -Recurse
+Copy-Item $CWD\..\..\src\nasui\build $CWD\nni\nasui -Recurse
+Copy-Item $CWD\..\..\src\nasui\server.js $CWD\nni\nasui -Recurse
 Copy-Item $CWD\..\..\src\nni_manager\package.json $CWD\nni
 (Get-Content $CWD\nni\package.json).replace($NNI_VERSION_TEMPLATE, $NNI_VERSION_VALUE) | Set-Content $CWD\nni\package.json
 cd $CWD\nni

install.ps1

@@ -116,6 +116,10 @@ Copy-Item config -Destination .\dist\ -Recurse -Force
 cd ..\webui
 cmd /c $NNI_YARN
 cmd /c $NNI_YARN build
+# Building NasUI
+cd ..\nasui
+cmd /c $NNI_YARN
+cmd /c $NNI_YARN build
 
 cd ..\..
 
@@ -130,4 +134,7 @@ $PKG_JSON = $NNI_PKG_FOLDER + "\package.json"
 (Get-Content $PKG_JSON).replace($NNI_VERSION_TEMPLATE, $NNI_VERSION_VALUE) | Set-Content $PKG_JSON
 cmd /c $NNI_YARN --prod --cwd $NNI_PKG_FOLDER
 $NNI_PKG_FOLDER_STATIC = $NNI_PKG_FOLDER + "\static"
+$NASUI_PKG_FOLDER = $NNI_PKG_FOLDER + "\nasui"
 Copy-Item "src\webui\build" $NNI_PKG_FOLDER_STATIC -Recurse
+Copy-Item "src\nasui\build" $NASUI_PKG_FOLDER -Recurse
+Copy-Item "src\nasui\server.js" $NASUI_PKG_FOLDER -Recurse

tools/nni_cmd/common_utils.py

@@ -2,6 +2,7 @@
 # Licensed under the MIT license.
 
 import os
+import site
 import sys
 import json
 import socket
@@ -74,3 +75,44 @@ def get_python_dir(sitepackages_path):
         return str(Path(sitepackages_path))
     else:
         return str(Path(sitepackages_path).parents[2])
+
+def get_nni_installation_path():
+    ''' Find nni lib from the following locations in order
+    Return nni root directory if it exists
+    '''
+    def try_installation_path_sequentially(*sitepackages):
+        '''Try different installation path sequentially util nni is found.
+        Return None if nothing is found
+        '''
+        def _generate_installation_path(sitepackages_path):
+            python_dir = get_python_dir(sitepackages_path)
+            entry_file = os.path.join(python_dir, 'nni', 'main.js')
+            if os.path.isfile(entry_file):
+                return python_dir
+            return None
+
+        for sitepackage in sitepackages:
+            python_dir = _generate_installation_path(sitepackage)
+            if python_dir:
+                return python_dir
+        return None
+
+    if os.getenv('VIRTUAL_ENV'):
+        # if 'virtualenv' package is used, `site` has not attr getsitepackages, so we will instead use VIRTUAL_ENV
+        # Note that conda venv will not have VIRTUAL_ENV
+        python_dir = os.getenv('VIRTUAL_ENV')
+    else:
+        python_sitepackage = site.getsitepackages()[0]
+        # If system-wide python is used, we will give priority to using `local sitepackage`--"usersitepackages()" given
+        # that nni exists there
+        if python_sitepackage.startswith('/usr') or python_sitepackage.startswith('/Library'):
+            python_dir = try_installation_path_sequentially(site.getusersitepackages(), site.getsitepackages()[0])
+        else:
+            python_dir = try_installation_path_sequentially(site.getsitepackages()[0], site.getusersitepackages())
+
+    if python_dir:
+        entry_file = os.path.join(python_dir, 'nni', 'main.js')
+        if os.path.isfile(entry_file):
+            return os.path.join(python_dir, 'nni')
+    print_error('Fail to find nni under python library')
+    exit(1)
\ No newline at end of file

tools/nni_cmd/launcher.py

@@ -6,7 +6,6 @@ import os
 import sys
 import string
 import random
-import site
 import time
 import tempfile
 from subprocess import Popen, check_call, CalledProcessError, PIPE, STDOUT
@@ -17,7 +16,7 @@ from .rest_utils import rest_put, rest_post, check_rest_server, check_response
 from .url_utils import cluster_metadata_url, experiment_url, get_local_urls
 from .config_utils import Config, Experiments
 from .common_utils import get_yml_content, get_json_content, print_error, print_normal, \
-                          detect_port, get_user, get_python_dir
+                          detect_port, get_user, get_nni_installation_path
 from .constants import NNICTL_HOME_DIR, ERROR_INFO, REST_TIME_OUT, EXPERIMENT_SUCCESS_INFO, LOG_HEADER, PACKAGE_REQUIREMENTS
 from .command_utils import check_output_command, kill_command
 from .nnictl_utils import update_experiment
@@ -37,47 +36,6 @@ def print_log_content(config_file_name):
     print_normal(' Stderr:')
     print(check_output_command(stderr_full_path))
 
-def get_nni_installation_path():
-    ''' Find nni lib from the following locations in order
-    Return nni root directory if it exists
-    '''
-    def try_installation_path_sequentially(*sitepackages):
-        '''Try different installation path sequentially util nni is found.
-        Return None if nothing is found
-        '''
-        def _generate_installation_path(sitepackages_path):
-            python_dir = get_python_dir(sitepackages_path)
-            entry_file = os.path.join(python_dir, 'nni', 'main.js')
-            if os.path.isfile(entry_file):
-                return python_dir
-            return None
-
-        for sitepackage in sitepackages:
-            python_dir = _generate_installation_path(sitepackage)
-            if python_dir:
-                return python_dir
-        return None
-
-    if os.getenv('VIRTUAL_ENV'):
-        # if 'virtualenv' package is used, `site` has not attr getsitepackages, so we will instead use VIRTUAL_ENV
-        # Note that conda venv will not have VIRTUAL_ENV
-        python_dir = os.getenv('VIRTUAL_ENV')
-    else:
-        python_sitepackage = site.getsitepackages()[0]
-        # If system-wide python is used, we will give priority to using `local sitepackage`--"usersitepackages()" given
-        # that nni exists there
-        if python_sitepackage.startswith('/usr') or python_sitepackage.startswith('/Library'):
-            python_dir = try_installation_path_sequentially(site.getusersitepackages(), site.getsitepackages()[0])
-        else:
-            python_dir = try_installation_path_sequentially(site.getsitepackages()[0], site.getusersitepackages())
-
-    if python_dir:
-        entry_file = os.path.join(python_dir, 'nni', 'main.js')
-        if os.path.isfile(entry_file):
-            return os.path.join(python_dir, 'nni')
-    print_error('Fail to find nni under python library')
-    exit(1)
-
 def start_rest_server(port, platform, mode, config_file_name, foreground=False, experiment_id=None, log_dir=None, log_level=None):
     '''Run nni manager process'''
     if detect_port(port):

tools/nni_cmd/nnictl_utils.py

@@ -19,7 +19,7 @@ from .url_utils import trial_jobs_url, experiment_url, trial_job_id_url, export_
 from .config_utils import Config, Experiments
 from .constants import NNICTL_HOME_DIR, EXPERIMENT_INFORMATION_FORMAT, EXPERIMENT_DETAIL_FORMAT, \
      EXPERIMENT_MONITOR_INFO, TRIAL_MONITOR_HEAD, TRIAL_MONITOR_CONTENT, TRIAL_MONITOR_TAIL, REST_TIME_OUT
-from .common_utils import print_normal, print_error, print_warning, detect_process, get_yml_content
+from .common_utils import print_normal, print_error, print_warning, detect_process, get_yml_content, get_nni_installation_path
 from .command_utils import check_output_command, kill_command
 from .ssh_utils import create_ssh_sftp_client, remove_remote_directory
 
@@ -392,10 +392,13 @@ def webui_url(args):
 def webui_nas(args):
     '''launch nas ui'''
     print_normal('Starting NAS UI...')
-    # TODO: find file path on installing with pypi
-    # TODO: use correct node on win32
     try:
-        cmds = ['node', 'src/nasui/server.js', '--port', str(args.port), '--logdir', args.logdir]
+        entry_dir = get_nni_installation_path()
+        entry_file = os.path.join(entry_dir, 'nasui', 'server.js')
+        node_command = 'node'
+        if sys.platform == 'win32':
+            node_command = os.path.join(entry_dir[:-3], 'Scripts', 'node.exe')
+        cmds = [node_command, '--max-old-space-size=4096', entry_file, '--port', str(args.port), '--logdir', args.logdir]
         subprocess.run(cmds)
     except KeyboardInterrupt:
         pass