Merge pull request #3444 from microsoft/v2.1

V2.1 merge back to master

nni/retiarii/strategy/utils.py

+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
 import collections
 from typing import Dict, Any, List
 from ..graph import Model

nni/retiarii/trial_entry.py

+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
 """
 Entrypoint for trials.
 

nni/retiarii/utils.py

+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
 import inspect
 import warnings
 from collections import defaultdict

nni/tools/nnictl/launcher.py

@@ -3,6 +3,7 @@
 
 import json
 import os
+from pathlib import Path
 import sys
 import string
 import random
@@ -590,14 +591,14 @@ def create_experiment(args):
     except Exception:
         print_warning('Validation with V1 schema failed. Trying to convert from V2 format...')
         try:
-            config = ExperimentConfig(**experiment_config)
+            config = ExperimentConfig(_base_path=Path(config_path).parent, **experiment_config)
             experiment_config = convert.to_v1_yaml(config)
         except Exception as e:
-            print_error(f'Conversion from v2 format failed: {repr(e)}')
+            print_error(f'Config in v2 format validation failed, the config error in v2 format is: {repr(e)}')
         try:
             validate_all_content(experiment_config, config_path)
         except Exception as e:
-            print_error(f'Config in v1 format validation failed. {repr(e)}')
+            print_error(f'Config in v1 format validation failed, the config error in v1 format is: {repr(e)}')
             exit(1)
 
     try:

test/config/assessors/curvefitting.yml

 authorName: nni
 experimentName: default_test
-maxExecDuration: 5m
+maxExecDuration: 10m
 maxTrialNum: 8
 trialConcurrency: 8
 searchSpacePath: ../naive_trial/search_space.json

test/nni_test/nnitest/run_tests.py

@@ -260,9 +260,9 @@ def run(args):
             continue
         # remote mode need more time to cleanup 
         if args.ts == 'remote':
-            wait_for_port_available(8080, 180)
+            wait_for_port_available(8080, 240)
         else:
-            wait_for_port_available(8080, 30)
+            wait_for_port_available(8080, 60)
 
         # adl mode need more time to cleanup PVC
         if args.ts == 'adl' and name == 'nnictl-resume-2':

ts/nni_manager/core/nnimanager.ts

@@ -326,22 +326,26 @@ class NNIManager implements Manager {
     }
 
     public async stopExperimentBottomHalf(): Promise<void> {
-        const trialJobList: TrialJobDetail[] = await this.trainingService.listTrialJobs();
-
-        // DON'T try to make it in parallel, the training service may not handle it well.
-        // If there is performance concern, consider to support batch cancellation on training service.
-        for (const trialJob of trialJobList) {
-            if (trialJob.status === 'RUNNING' ||
-                trialJob.status === 'WAITING') {
-                try {
-                    this.log.info(`cancelTrialJob: ${trialJob.id}`);
-                    await this.trainingService.cancelTrialJob(trialJob.id);
-                } catch (error) {
-                    this.log.debug(`ignorable error on canceling trial ${trialJob.id}. ${error}`);
+        try {
+            const trialJobList: TrialJobDetail[] = await this.trainingService.listTrialJobs();
+
+            // DON'T try to make it in parallel, the training service may not handle it well.
+            // If there is performance concern, consider to support batch cancellation on training service.
+            for (const trialJob of trialJobList) {
+                if (trialJob.status === 'RUNNING' ||
+                    trialJob.status === 'WAITING') {
+                    try {
+                        this.log.info(`cancelTrialJob: ${trialJob.id}`);
+                        await this.trainingService.cancelTrialJob(trialJob.id);
+                    } catch (error) {
+                        this.log.debug(`ignorable error on canceling trial ${trialJob.id}. ${error}`);
+                    }
                 }
             }
+            await this.trainingService.cleanUp();
+        } catch (err) {
+            this.log.error(`${err.stack}`);
         }
-        await this.trainingService.cleanUp();
         if (this.experimentProfile.endTime === undefined) {
             this.setEndtime();
         }