Commit 8c07cf41 authored by Seung Ho Jang's avatar Seung Ho Jang Committed by chicm-ms
Browse files

Add support for zero gpuNum in kubernetes (#1830)

parent 97b258b0
...@@ -64,11 +64,16 @@ abstract class KubernetesTrainingService { ...@@ -64,11 +64,16 @@ abstract class KubernetesTrainingService {
// tslint:disable:no-any // tslint:disable:no-any
public generatePodResource(memory: number, cpuNum: number, gpuNum: number): any { public generatePodResource(memory: number, cpuNum: number, gpuNum: number): any {
return { const resources: any = {
memory: `${memory}Mi`, memory: `${memory}Mi`,
cpu: `${cpuNum}`, cpu: `${cpuNum}`
'nvidia.com/gpu': `${gpuNum}`
}; };
if (gpuNum !== 0) {
resources['nvidia.com/gpu'] = `${gpuNum}`;
}
return resources;
} // tslint:enable:no-any } // tslint:enable:no-any
public async listTrialJobs(): Promise<TrialJobDetail[]> { public async listTrialJobs(): Promise<TrialJobDetail[]> {
...@@ -265,7 +270,7 @@ abstract class KubernetesTrainingService { ...@@ -265,7 +270,7 @@ abstract class KubernetesTrainingService {
// Refer https://github.com/NVIDIA/k8s-device-plugin/issues/61 // Refer https://github.com/NVIDIA/k8s-device-plugin/issues/61
// So we have to explicitly set CUDA_VISIBLE_DEVICES to empty if user sets gpuNum to 0 in NNI config file // So we have to explicitly set CUDA_VISIBLE_DEVICES to empty if user sets gpuNum to 0 in NNI config file
if (gpuNum === 0) { if (gpuNum === 0) {
nvidiaScript = `export CUDA_VISIBLE_DEVICES='0'`; nvidiaScript = 'export CUDA_VISIBLE_DEVICES=';
} }
// tslint:disable-next-line: strict-boolean-expressions // tslint:disable-next-line: strict-boolean-expressions
const nniManagerIp: string = this.nniManagerIpConfig ? this.nniManagerIpConfig.nniManagerIp : getIPV4Address(); const nniManagerIp: string = this.nniManagerIpConfig ? this.nniManagerIpConfig.nniManagerIp : getIPV4Address();
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment