Fix remoteTrainingService gpuScheduler (#1749)

src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts

@@ -67,7 +67,7 @@ class RemoteMachineTrainingService implements TrainingService {
     private readonly expRootDir: string;
     private readonly remoteExpRootDir: string;
     private trialConfig: TrialConfig | undefined;
-    private readonly gpuScheduler: GPUScheduler;
+    private gpuScheduler?: GPUScheduler;
     private readonly jobQueue: string[];
     private readonly timer: ObservableTimer;
     private stopping: boolean = false;
@@ -87,7 +87,6 @@ class RemoteMachineTrainingService implements TrainingService {
         this.trialJobsMap = new Map<string, RemoteMachineTrialJobDetail>();
         this.trialSSHClientMap = new Map<string, Client>();
         this.machineSSHClientMap = new Map<RemoteMachineMeta, SSHClientManager>();
-        this.gpuScheduler = new GPUScheduler(this.machineSSHClientMap);
         this.jobQueue = [];
         this.expRootDir = getExperimentRootDir();
         this.remoteExpRootDir = this.getRemoteExperimentRootDir();
@@ -334,6 +333,7 @@ class RemoteMachineTrainingService implements TrainingService {
                 break;
             case TrialConfigMetadataKey.MACHINE_LIST:
                 await this.setupConnections(value);
+                this.gpuScheduler = new GPUScheduler(this.machineSSHClientMap);
                 break;
             case TrialConfigMetadataKey.TRIAL_CONFIG:
                 const remoteMachineTrailConfig: TrialConfig = <TrialConfig>JSON.parse(value);
@@ -397,12 +397,14 @@ class RemoteMachineTrainingService implements TrainingService {
      * remove gpu reversion when job is not running
      */
     private updateGpuReservation(): void {
+        if (this.gpuScheduler) {
             for (const [key, value] of this.trialJobsMap) {
                 if (!['WAITING', 'RUNNING'].includes(value.status)) {
                     this.gpuScheduler.removeGpuReservation(key, this.trialJobsMap);
                 }
             }
         }
+    }
 
     /**
      * stop gpu_metric_collector process in remote machine and remove unused scripts
@@ -483,6 +485,9 @@ class RemoteMachineTrainingService implements TrainingService {
         if (this.trialConfig === undefined) {
             throw new Error('trial config is not initialized');
         }
+        if (this.gpuScheduler === undefined) {
+            throw new Error('gpuScheduler is not initialized');
+        }
         const trialJobDetail: RemoteMachineTrialJobDetail | undefined = this.trialJobsMap.get(trialJobId);
         if (trialJobDetail === undefined) {
             throw new NNIError(NNIErrorNames.INVALID_JOB_DETAIL, `Invalid job detail information for trial job ${trialJobId}`);