Unverified Commit 715b1899 authored by SparkSnail's avatar SparkSnail Committed by GitHub
Browse files

Fix reuse mode env folder (#3374)

parent 0d37a64a
...@@ -113,7 +113,7 @@ export class AMLEnvironmentService extends EnvironmentService { ...@@ -113,7 +113,7 @@ export class AMLEnvironmentService extends EnvironmentService {
throw new Error('AML trial config is not initialized'); throw new Error('AML trial config is not initialized');
} }
const amlEnvironment: AMLEnvironmentInformation = environment as AMLEnvironmentInformation; const amlEnvironment: AMLEnvironmentInformation = environment as AMLEnvironmentInformation;
const environmentLocalTempFolder = path.join(this.experimentRootDir, this.experimentId, "environment-temp"); const environmentLocalTempFolder = path.join(this.experimentRootDir, "environment-temp");
environment.command = `import os\nos.system('mv envs outputs/envs && cd outputs && ${amlEnvironment.command}')`; environment.command = `import os\nos.system('mv envs outputs/envs && cd outputs && ${amlEnvironment.command}')`;
environment.useActiveGpu = this.amlClusterConfig.useActiveGpu; environment.useActiveGpu = this.amlClusterConfig.useActiveGpu;
environment.maxTrialNumberPerGpu = this.amlClusterConfig.maxTrialNumPerGpu; environment.maxTrialNumberPerGpu = this.amlClusterConfig.maxTrialNumPerGpu;
......
...@@ -101,7 +101,7 @@ export class LocalEnvironmentService extends EnvironmentService { ...@@ -101,7 +101,7 @@ export class LocalEnvironmentService extends EnvironmentService {
`Write $LASTEXITCODE " " $NOW_DATE | Out-File "${path.join(environment.runnerWorkingFolder, 'code')}" -NoNewline -encoding utf8`); `Write $LASTEXITCODE " " $NOW_DATE | Out-File "${path.join(environment.runnerWorkingFolder, 'code')}" -NoNewline -encoding utf8`);
} else { } else {
script.push(`cd ${this.experimentRootDir}`); script.push(`cd ${this.experimentRootDir}`);
script.push(`eval ${environment.command} --job_pid_file ${environment.runnerWorkingFolder}/pid 1>${environment.runnerWorkingFolder}/trialrunner_stdout 2>${environment.runnerWorkingFolder}/trialrunner_stderr"`); script.push(`eval ${environment.command} --job_pid_file ${environment.runnerWorkingFolder}/pid 1>${environment.runnerWorkingFolder}/trialrunner_stdout 2>${environment.runnerWorkingFolder}/trialrunner_stderr`);
if (process.platform === 'darwin') { if (process.platform === 'darwin') {
// https://superuser.com/questions/599072/how-to-get-bash-execution-time-in-milliseconds-under-mac-os-x // https://superuser.com/questions/599072/how-to-get-bash-execution-time-in-milliseconds-under-mac-os-x
// Considering the worst case, write 999 to avoid negative duration // Considering the worst case, write 999 to avoid negative duration
...@@ -118,9 +118,7 @@ export class LocalEnvironmentService extends EnvironmentService { ...@@ -118,9 +118,7 @@ export class LocalEnvironmentService extends EnvironmentService {
if (this.localTrialConfig === undefined) { if (this.localTrialConfig === undefined) {
throw new Error('Local trial config is not initialized'); throw new Error('Local trial config is not initialized');
} }
// Need refactor, this temp folder path is not appropriate, there are two expId in this path const localTempFolder: string = path.join(this.experimentRootDir, "environment-temp", "envs");
const localTempFolder: string = path.join(this.experimentRootDir, this.experimentId,
"environment-temp", "envs");
const localEnvCodeFolder: string = path.join(this.experimentRootDir, "envs"); const localEnvCodeFolder: string = path.join(this.experimentRootDir, "envs");
environment.runnerWorkingFolder = path.join(localEnvCodeFolder, environment.id); environment.runnerWorkingFolder = path.join(localEnvCodeFolder, environment.id);
await execMkdir(environment.runnerWorkingFolder); await execMkdir(environment.runnerWorkingFolder);
......
...@@ -264,7 +264,7 @@ ${environment.command} --job_pid_file ${environment.runnerWorkingFolder}/pid \ ...@@ -264,7 +264,7 @@ ${environment.command} --job_pid_file ${environment.runnerWorkingFolder}/pid \
} }
const executor = await this.getExecutor(environment.id); const executor = await this.getExecutor(environment.id);
const environmentLocalTempFolder: string = const environmentLocalTempFolder: string =
path.join(this.experimentRootDir, this.experimentId, "environment-temp") path.join(this.experimentRootDir, "environment-temp")
await executor.createFolder(environment.runnerWorkingFolder); await executor.createFolder(environment.runnerWorkingFolder);
await execMkdir(environmentLocalTempFolder); await execMkdir(environmentLocalTempFolder);
await fs.promises.writeFile(path.join(environmentLocalTempFolder, executor.getScriptName("run")), await fs.promises.writeFile(path.join(environmentLocalTempFolder, executor.getScriptName("run")),
......
...@@ -201,7 +201,7 @@ class TrialDispatcher implements TrainingService { ...@@ -201,7 +201,7 @@ class TrialDispatcher implements TrainingService {
} else { } else {
this.log.debug(`TrialDispatcher: create temp storage service to temp folder.`); this.log.debug(`TrialDispatcher: create temp storage service to temp folder.`);
storageService = new MountedStorageService(); storageService = new MountedStorageService();
const environmentLocalTempFolder = path.join(this.experimentRootDir, this.experimentId, "environment-temp"); const environmentLocalTempFolder = path.join(this.experimentRootDir, "environment-temp");
storageService.initialize(this.trialConfig.codeDir, environmentLocalTempFolder); storageService.initialize(this.trialConfig.codeDir, environmentLocalTempFolder);
} }
// Copy the compressed file to remoteDirectory and delete it // Copy the compressed file to remoteDirectory and delete it
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment