Unverified Commit 715b1899 authored by SparkSnail's avatar SparkSnail Committed by GitHub
Browse files

Fix reuse mode env folder (#3374)

parent 0d37a64a
......@@ -113,7 +113,7 @@ export class AMLEnvironmentService extends EnvironmentService {
throw new Error('AML trial config is not initialized');
}
const amlEnvironment: AMLEnvironmentInformation = environment as AMLEnvironmentInformation;
const environmentLocalTempFolder = path.join(this.experimentRootDir, this.experimentId, "environment-temp");
const environmentLocalTempFolder = path.join(this.experimentRootDir, "environment-temp");
environment.command = `import os\nos.system('mv envs outputs/envs && cd outputs && ${amlEnvironment.command}')`;
environment.useActiveGpu = this.amlClusterConfig.useActiveGpu;
environment.maxTrialNumberPerGpu = this.amlClusterConfig.maxTrialNumPerGpu;
......
......@@ -101,7 +101,7 @@ export class LocalEnvironmentService extends EnvironmentService {
`Write $LASTEXITCODE " " $NOW_DATE | Out-File "${path.join(environment.runnerWorkingFolder, 'code')}" -NoNewline -encoding utf8`);
} else {
script.push(`cd ${this.experimentRootDir}`);
script.push(`eval ${environment.command} --job_pid_file ${environment.runnerWorkingFolder}/pid 1>${environment.runnerWorkingFolder}/trialrunner_stdout 2>${environment.runnerWorkingFolder}/trialrunner_stderr"`);
script.push(`eval ${environment.command} --job_pid_file ${environment.runnerWorkingFolder}/pid 1>${environment.runnerWorkingFolder}/trialrunner_stdout 2>${environment.runnerWorkingFolder}/trialrunner_stderr`);
if (process.platform === 'darwin') {
// https://superuser.com/questions/599072/how-to-get-bash-execution-time-in-milliseconds-under-mac-os-x
// Considering the worst case, write 999 to avoid negative duration
......@@ -118,9 +118,7 @@ export class LocalEnvironmentService extends EnvironmentService {
if (this.localTrialConfig === undefined) {
throw new Error('Local trial config is not initialized');
}
// Need refactor, this temp folder path is not appropriate, there are two expId in this path
const localTempFolder: string = path.join(this.experimentRootDir, this.experimentId,
"environment-temp", "envs");
const localTempFolder: string = path.join(this.experimentRootDir, "environment-temp", "envs");
const localEnvCodeFolder: string = path.join(this.experimentRootDir, "envs");
environment.runnerWorkingFolder = path.join(localEnvCodeFolder, environment.id);
await execMkdir(environment.runnerWorkingFolder);
......
......@@ -264,7 +264,7 @@ ${environment.command} --job_pid_file ${environment.runnerWorkingFolder}/pid \
}
const executor = await this.getExecutor(environment.id);
const environmentLocalTempFolder: string =
path.join(this.experimentRootDir, this.experimentId, "environment-temp")
path.join(this.experimentRootDir, "environment-temp")
await executor.createFolder(environment.runnerWorkingFolder);
await execMkdir(environmentLocalTempFolder);
await fs.promises.writeFile(path.join(environmentLocalTempFolder, executor.getScriptName("run")),
......
......@@ -201,7 +201,7 @@ class TrialDispatcher implements TrainingService {
} else {
this.log.debug(`TrialDispatcher: create temp storage service to temp folder.`);
storageService = new MountedStorageService();
const environmentLocalTempFolder = path.join(this.experimentRootDir, this.experimentId, "environment-temp");
const environmentLocalTempFolder = path.join(this.experimentRootDir, "environment-temp");
storageService.initialize(this.trialConfig.codeDir, environmentLocalTempFolder);
}
// Copy the compressed file to remoteDirectory and delete it
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment