Fix gpu detector in localTrainingService (#1068)

af89df8c · SparkSnail · GitHub · 5af01545 · af89df8c
Unverified Commit af89df8c authored May 16, 2019 by SparkSnail Committed by GitHub May 16, 2019
Show whitespace changes
Inline Side-by-side

Showing with 10 additions and 6 deletions

src/nni_manager/training_service/local/gpuScheduler.ts src/nni_manager/training_service/local/gpuScheduler.ts +10 -6

No files found.
--- a/src/nni_manager/training_service/local/gpuScheduler.ts
+++ b/src/nni_manager/training_service/local/gpuScheduler.ts
@@ -105,13 +105,17 @@ class GPUScheduler {
    }
    private async updateGPUSummary(): Promise<void> {
-        const cmdresult: cpp.childProcessPromise.Result =
+        let gpuMetricPath = path.join(this.gpuMetricCollectorScriptFolder, 'gpu_metrics');
-            await execTail(path.join(this.gpuMetricCollectorScriptFolder, 'gpu_metrics'));
+        if (fs.existsSync(gpuMetricPath)) {
+            const cmdresult: cpp.childProcessPromise.Result = await execTail(gpuMetricPath);
            if (cmdresult && cmdresult.stdout) {
                this.gpuSummary = <GPUSummary>JSON.parse(cmdresult.stdout);
            } else {
                this.log.error('Could not get gpu metrics information!');
            }
+        } else{
+            this.log.warning('gpu_metrics file does not exist!')
+        }
    }
 }