Unverified Commit 684005d7 authored by Ni Hao's avatar Ni Hao Committed by GitHub
Browse files

fix no log from subprocess on trial (#3653)


Co-authored-by: default avatarHao Ni <v-nihao@microsoft.com>
parent 277e63f2
...@@ -8,7 +8,7 @@ ...@@ -8,7 +8,7 @@
*/ */
type TrialJobStatus = 'UNKNOWN' | 'WAITING' | 'RUNNING' | 'SUCCEEDED' | 'FAILED' | 'USER_CANCELED' | 'SYS_CANCELED' | 'EARLY_STOPPED'; type TrialJobStatus = 'UNKNOWN' | 'WAITING' | 'RUNNING' | 'SUCCEEDED' | 'FAILED' | 'USER_CANCELED' | 'SYS_CANCELED' | 'EARLY_STOPPED';
type LogType = 'TRIAL_LOG' | 'TRIAL_ERROR'; type LogType = 'TRIAL_LOG' | 'TRIAL_STDOUT' | 'TRIAL_ERROR';
interface TrainingServiceMetadata { interface TrainingServiceMetadata {
readonly key: string; readonly key: string;
......
...@@ -174,6 +174,8 @@ class LocalTrainingService implements TrainingService { ...@@ -174,6 +174,8 @@ class LocalTrainingService implements TrainingService {
let logPath: string; let logPath: string;
if (logType === 'TRIAL_LOG') { if (logType === 'TRIAL_LOG') {
logPath = path.join(this.rootDir, 'trials', trialJobId, 'trial.log'); logPath = path.join(this.rootDir, 'trials', trialJobId, 'trial.log');
} else if (logType === 'TRIAL_STDOUT'){
logPath = path.join(this.rootDir, 'trials', trialJobId, 'stdout');
} else if (logType === 'TRIAL_ERROR') { } else if (logType === 'TRIAL_ERROR') {
logPath = path.join(this.rootDir, 'trials', trialJobId, 'stderr'); logPath = path.join(this.rootDir, 'trials', trialJobId, 'stderr');
} else { } else {
...@@ -412,15 +414,16 @@ class LocalTrainingService implements TrainingService { ...@@ -412,15 +414,16 @@ class LocalTrainingService implements TrainingService {
private getScript(workingDirectory: string): string[] { private getScript(workingDirectory: string): string[] {
const script: string[] = []; const script: string[] = [];
if (process.platform === 'win32') { if (process.platform === 'win32') {
script.push(`$PSDefaultParameterValues = @{'Out-File:Encoding' = 'utf8'}`);
script.push(`cd $env:NNI_CODE_DIR`); script.push(`cd $env:NNI_CODE_DIR`);
script.push( script.push(
`cmd.exe /c ${this.config.trialCommand} 2>&1 | Out-File "${path.join(workingDirectory, 'stderr')}" -encoding utf8`, `cmd.exe /c ${this.config.trialCommand} 1>${path.join(workingDirectory, 'stdout')} 2>${path.join(workingDirectory, 'stderr')}`,
`$NOW_DATE = [int64](([datetime]::UtcNow)-(get-date "1/1/1970")).TotalSeconds`, `$NOW_DATE = [int64](([datetime]::UtcNow)-(get-date "1/1/1970")).TotalSeconds`,
`$NOW_DATE = "$NOW_DATE" + (Get-Date -Format fff).ToString()`, `$NOW_DATE = "$NOW_DATE" + (Get-Date -Format fff).ToString()`,
`Write $LASTEXITCODE " " $NOW_DATE | Out-File "${path.join(workingDirectory, '.nni', 'state')}" -NoNewline -encoding utf8`); `Write $LASTEXITCODE " " $NOW_DATE | Out-File "${path.join(workingDirectory, '.nni', 'state')}" -NoNewline -encoding utf8`);
} else { } else {
script.push(`cd $NNI_CODE_DIR`); script.push(`cd $NNI_CODE_DIR`);
script.push(`eval ${this.config.trialCommand} 2>"${path.join(workingDirectory, 'stderr')}"`); script.push(`eval ${this.config.trialCommand} 1>${path.join(workingDirectory, 'stdout')} 2>${path.join(workingDirectory, 'stderr')}`);
if (process.platform === 'darwin') { if (process.platform === 'darwin') {
// https://superuser.com/questions/599072/how-to-get-bash-execution-time-in-milliseconds-under-mac-os-x // https://superuser.com/questions/599072/how-to-get-bash-execution-time-in-milliseconds-under-mac-os-x
// Considering the worst case, write 999 to avoid negative duration // Considering the worst case, write 999 to avoid negative duration
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment