Unverified Commit 392c598e authored by fishyds's avatar fishyds Committed by GitHub
Browse files

Change hard-coded root directory to $PWD in PAI container (#182)

parent 93546854
...@@ -133,10 +133,13 @@ export namespace HDFSClientUtility { ...@@ -133,10 +133,13 @@ export namespace HDFSClientUtility {
deferred.resolve(exist); deferred.resolve(exist);
}); });
// Set timeout and reject the promise once reach timeout (5 seconds) let timeoutId : NodeJS.Timer
setTimeout(() => deferred.reject(`Check HDFS path ${hdfsPath} exists timeout`), 5000); const delayTimeout : Promise<boolean> = new Promise<boolean>((resolve : Function, reject : Function) : void => {
// Set timeout and reject the promise once reach timeout (5 seconds)
setTimeout(() => deferred.reject(`Check HDFS path ${hdfsPath} exists timeout`), 5000);
});
return deferred.promise; return Promise.race([deferred.promise, delayTimeout]).finally(() => clearTimeout(timeoutId));
} }
/** /**
......
...@@ -186,8 +186,8 @@ class PAITrainingService implements TrainingService { ...@@ -186,8 +186,8 @@ class PAITrainingService implements TrainingService {
const nniPaiTrialCommand : string = String.Format( const nniPaiTrialCommand : string = String.Format(
PAI_TRIAL_COMMAND_FORMAT, PAI_TRIAL_COMMAND_FORMAT,
// PAI will copy job's codeDir into /root directory // PAI will copy job's codeDir into /root directory
`/root/${trialJobId}`, `$PWD/${trialJobId}`,
`/root/${trialJobId}/nnioutput`, `$PWD/${trialJobId}/nnioutput`,
trialJobId, trialJobId,
this.experimentId, this.experimentId,
this.paiTrialConfig.command, this.paiTrialConfig.command,
...@@ -343,7 +343,17 @@ class PAITrainingService implements TrainingService { ...@@ -343,7 +343,17 @@ class PAITrainingService implements TrainingService {
deferred.resolve(); deferred.resolve();
} }
}); });
break;
let timeoutId: NodeJS.Timer;
const timeoutDelay: Promise<void> = new Promise<void>((resolve: Function, reject: Function): void => {
// Set timeout and reject the promise once reach timeout (5 seconds)
timeoutId = setTimeout(
() => reject(new Error('Get PAI token timeout. Please check your PAI cluster.')),
5000);
});
return Promise.race([timeoutDelay, deferred.promise]).finally(() => clearTimeout(timeoutId));
case TrialConfigMetadataKey.TRIAL_CONFIG: case TrialConfigMetadataKey.TRIAL_CONFIG:
if (!this.paiClusterConfig){ if (!this.paiClusterConfig){
this.log.error('pai cluster config is not initialized'); this.log.error('pai cluster config is not initialized');
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment