Merge pull request #3023 from microsoft/v1.9

[do not squash!] merge v1.9 back to master

test/nni_test/nnitest/generate_ts_config.py

@@ -86,6 +86,8 @@ def update_training_service_config(args):
             config[args.ts]['machineList'][0]['port'] = args.remote_port
         if args.remote_pwd is not None:
             config[args.ts]['machineList'][0]['passwd'] = args.remote_pwd
+        if args.remote_reuse is not None:
+            config[args.ts]['remoteConfig']['reuse'] = args.remote_reuse.lower() == 'true'
 
     dump_yml_content(TRAINING_SERVICE_FILE, config)
 
@@ -119,6 +121,7 @@ if __name__ == '__main__':
     parser.add_argument("--remote_pwd", type=str)
     parser.add_argument("--remote_host", type=str)
     parser.add_argument("--remote_port", type=int)
+    parser.add_argument("--remote_reuse", type=str)
     args = parser.parse_args()
 
     update_training_service_config(args)

test/pipelines/pipelines-it-remote-linux-to-linux.yml

@@ -62,7 +62,7 @@ jobs:
   - script: |
       set -e
       cd test
-      python3 nni_test/nnitest/generate_ts_config.py --ts remote --remote_user $(docker_user) --remote_host $(remote_host) \
+      python3 nni_test/nnitest/generate_ts_config.py --ts remote --remote_reuse $(remote_reuse) --remote_user $(docker_user) --remote_host $(remote_host) \
       --remote_port $(cat port) --remote_pwd $(docker_pwd) --nni_manager_ip $(nni_manager_ip)
       cat config/training_service.yml
       PATH=$HOME/.local/bin:$PATH python3 nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts remote

test/pipelines/pipelines-it-remote-windows-to-linux.yml

@@ -48,7 +48,7 @@ jobs:
     displayName: 'Get docker port'
   - powershell: |
       cd test
-      python nni_test/nnitest/generate_ts_config.py --ts remote --remote_user $(docker_user) --remote_host $(remote_host) --remote_port $(Get-Content port) --remote_pwd $(docker_pwd) --nni_manager_ip $(nni_manager_ip)
+      python nni_test/nnitest/generate_ts_config.py --ts remote --remote_reuse $(remote_reuse) --remote_user $(docker_user) --remote_host $(remote_host) --remote_port $(Get-Content port) --remote_pwd $(docker_pwd) --nni_manager_ip $(nni_manager_ip)
       Get-Content config/training_service.yml
       python nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts remote --exclude cifar10
     displayName: 'integration test'

tools/nni_annotation/test_annotation.py

@@ -5,12 +5,13 @@
 
 from .__init__ import *
 
+import sys
 import ast
 import json
 import os
 import shutil
 import tempfile
-from unittest import TestCase, main
+from unittest import TestCase, main, skipIf
 
 
 class AnnotationTestCase(TestCase):
@@ -26,6 +27,7 @@ class AnnotationTestCase(TestCase):
         with open('testcase/searchspace.json') as f:
             self.assertEqual(search_space, json.load(f))
 
+    @skipIf(sys.version_info.major == 3 and sys.version_info.minor > 7, "skip for python3.8 temporarily")
     def test_code_generator(self):
         code_dir = expand_annotations('testcase/usercode', '_generated/usercode', nas_mode='classic_mode')
         self.assertEqual(code_dir, '_generated/usercode')

tools/nni_cmd/launcher.py

@@ -371,6 +371,11 @@ def set_experiment(experiment_config, mode, port, config_file_name):
             {'key': 'frameworkcontroller_config', 'value': experiment_config['frameworkcontrollerConfig']})
         request_data['clusterMetaData'].append(
             {'key': 'trial_config', 'value': experiment_config['trial']})
+    elif experiment_config['trainingServicePlatform'] == 'aml':
+        request_data['clusterMetaData'].append(
+            {'key': 'aml_config', 'value': experiment_config['amlConfig']})
+        request_data['clusterMetaData'].append(
+            {'key': 'trial_config', 'value': experiment_config['trial']})
     response = rest_post(experiment_url(port), json.dumps(request_data), REST_TIME_OUT, show_error=True)
     if check_response(response):
         return response

tools/nni_trial_tool/trial.py

@@ -137,10 +137,15 @@ class Trial:
     def kill(self, trial_id=None):
         if trial_id == self.id or trial_id is None:
             if self.process is not None:
-                nni_log(LogType.Info, "%s: killing trial" % self.name)
-                for child in psutil.Process(self.process.pid).children(True):
-                    child.kill()
-                self.process.kill()
+                try:
+                    nni_log(LogType.Info, "%s: killing trial" % self.name)
+                    for child in psutil.Process(self.process.pid).children(True):
+                        child.kill()
+                    self.process.kill()
+                except psutil.NoSuchProcess:
+                    nni_log(LogType.Info, "kill trial %s failed: %s does not exist!" % (trial_id, self.process.pid))
+                except Exception as ex:
+                    nni_log(LogType.Error, "kill trial %s failed: %s " % (trial_id, str(ex)))
             self.cleanup()
 
     def cleanup(self):