fix aml outputs and python process not killed (#3321)

* fix outputs and python process not killed

* change cleanup stop environment logic

* fix bug

ts/nni_manager/training_service/reusable/environments/amlEnvironmentService.ts

@@ -114,7 +114,7 @@ export class AMLEnvironmentService extends EnvironmentService {
         }
         const amlEnvironment: AMLEnvironmentInformation = environment as AMLEnvironmentInformation;
         const environmentLocalTempFolder = path.join(this.experimentRootDir, this.experimentId, "environment-temp");
-        environment.command = `import os\nos.system('${amlEnvironment.command}')`;
+        environment.command = `import os\nos.system('mv envs outputs/envs && cd outputs && ${amlEnvironment.command}')`;
         environment.useActiveGpu = this.amlClusterConfig.useActiveGpu;
         environment.maxTrialNumberPerGpu = this.amlClusterConfig.maxTrialNumPerGpu;
         await fs.promises.writeFile(path.join(environmentLocalTempFolder, 'nni_script.py'), amlEnvironment.command, { encoding: 'utf8' });

ts/nni_manager/training_service/reusable/environments/localEnvironmentService.ts

@@ -111,6 +111,10 @@ ${environment.command} --job_pid_file ${environment.runnerWorkingFolder}/pid \
     }
 
     public async stopEnvironment(environment: EnvironmentInformation): Promise<void> {
+        if (environment.isAlive === false) {
+            return Promise.resolve();
+        }
+
         const jobpidPath: string = `${environment.runnerWorkingFolder}/pid`;
         const pid: string = await fs.promises.readFile(jobpidPath, 'utf8');
         tkill(Number(pid), 'SIGKILL');

ts/nni_manager/training_service/reusable/environments/openPaiEnvironmentService.ts

@@ -219,6 +219,9 @@ export class OpenPaiEnvironmentService extends EnvironmentService {
     public async stopEnvironment(environment: EnvironmentInformation): Promise<void> {
         const deferred: Deferred<void> = new Deferred<void>();
 
+        if (environment.isAlive === false) {
+            return Promise.resolve();
+        }
         if (this.paiClusterConfig === undefined) {
             return Promise.reject(new Error('PAI Cluster config is not initialized'));
         }

ts/nni_manager/training_service/reusable/environments/remoteEnvironmentService.ts

@@ -289,6 +289,10 @@ ${environment.command} --job_pid_file ${environment.runnerWorkingFolder}/pid \
     }
 
     public async stopEnvironment(environment: EnvironmentInformation): Promise<void> {
+        if (environment.isAlive === false) {
+            return Promise.resolve();
+        }
+
         const executor = await this.getExecutor(environment.id);
 
         if (environment.status === 'UNKNOWN') {

ts/nni_manager/training_service/reusable/trialDispatcher.ts

@@ -310,14 +310,12 @@ class TrialDispatcher implements TrainingService {
 
         for (let index = 0; index < environments.length; index++) {
             const environment = environments[index];
-            if (environment.isAlive === true) {
-                this.log.info(`stopping environment ${environment.id}...`);
-                if (environment.environmentService === undefined) {
-                    throw new Error(`${environment.id} do not have environmentService!`);
-                }
-                await environment.environmentService.stopEnvironment(environment);
-                this.log.info(`stopped environment ${environment.id}.`);
+            this.log.info(`stopping environment ${environment.id}...`);
+            if (environment.environmentService === undefined) {
+                throw new Error(`${environment.id} do not have environmentService!`);
             }
+            await environment.environmentService.stopEnvironment(environment);
+            this.log.info(`stopped environment ${environment.id}.`);
         }
         this.commandEmitter.off("command", this.handleCommand);
         for (const commandChannel of this.commandChannelSet) {