Commit 1df750e2 authored by Yan Ni's avatar Yan Ni Committed by QuanluZhang
Browse files

add gpuNum check for local TS (#378)

* add gpuNum check for local TS

* set CUDA_VISIBLE_DEVICES to empty string when gpuNum is 0

* remove redundency code
parent e341df81
......@@ -61,7 +61,7 @@ class LocalTrainingServiceForGPU extends LocalTrainingService {
this.requiredGPUNum = 0;
}
this.log.info('required GPU number is ' + this.requiredGPUNum);
if (this.gpuScheduler === undefined) {
if (this.gpuScheduler === undefined && this.requiredGPUNum > 0) {
this.gpuScheduler = new GPUScheduler();
}
break;
......@@ -78,7 +78,7 @@ class LocalTrainingServiceForGPU extends LocalTrainingService {
}
protected onTrialJobStatusChanged(trialJob: LocalTrialJobDetailForGPU, oldStatus: TrialJobStatus): void {
if (trialJob.gpuIndices !== undefined && trialJob.gpuIndices.length !== 0) {
if (trialJob.gpuIndices !== undefined && trialJob.gpuIndices.length !== 0 && this.gpuScheduler !== undefined) {
if (oldStatus === 'RUNNING' && trialJob.status !== 'RUNNING') {
for (const index of trialJob.gpuIndices) {
this.availableGPUIndices[index] = false;
......@@ -93,7 +93,7 @@ class LocalTrainingServiceForGPU extends LocalTrainingService {
const variables: { key: string; value: string }[] = super.getEnvironmentVariables(trialJobDetail, resource);
variables.push({
key: 'CUDA_VISIBLE_DEVICES',
value: resource.gpuIndices.join(',')
value: this.gpuScheduler === undefined ? '' : resource.gpuIndices.join(',')
});
return variables;
......@@ -125,8 +125,10 @@ class LocalTrainingServiceForGPU extends LocalTrainingService {
protected occupyResource(resource: { gpuIndices: number[] }): void {
super.occupyResource(resource);
for (const index of resource.gpuIndices) {
this.availableGPUIndices[index] = true;
if (this.gpuScheduler !== undefined) {
for (const index of resource.gpuIndices) {
this.availableGPUIndices[index] = true;
}
}
}
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment