Unverified Commit af89df8c authored by SparkSnail's avatar SparkSnail Committed by GitHub
Browse files

Fix gpu detector in localTrainingService (#1068)

parent 5af01545
......@@ -105,12 +105,16 @@ class GPUScheduler {
}
private async updateGPUSummary(): Promise<void> {
const cmdresult: cpp.childProcessPromise.Result =
await execTail(path.join(this.gpuMetricCollectorScriptFolder, 'gpu_metrics'));
if (cmdresult && cmdresult.stdout) {
this.gpuSummary = <GPUSummary>JSON.parse(cmdresult.stdout);
} else {
this.log.error('Could not get gpu metrics information!');
let gpuMetricPath = path.join(this.gpuMetricCollectorScriptFolder, 'gpu_metrics');
if (fs.existsSync(gpuMetricPath)) {
const cmdresult: cpp.childProcessPromise.Result = await execTail(gpuMetricPath);
if (cmdresult && cmdresult.stdout) {
this.gpuSummary = <GPUSummary>JSON.parse(cmdresult.stdout);
} else {
this.log.error('Could not get gpu metrics information!');
}
} else{
this.log.warning('gpu_metrics file does not exist!')
}
}
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment