Fix paiToken update logic (#976)

9a3c61e8 · SparkSnail · GitHub · 44f54866 · 9a3c61e8
Unverified Commit 9a3c61e8 authored Apr 17, 2019 by SparkSnail Committed by GitHub Apr 17, 2019
Hide whitespace changes
Inline Side-by-side

Showing with 9 additions and 1 deletion

src/nni_manager/training_service/pai/paiTrainingService.ts src/nni_manager/training_service/pai/paiTrainingService.ts +9 -1

No files found.
--- a/src/nni_manager/training_service/pai/paiTrainingService.ts
+++ b/src/nni_manager/training_service/pai/paiTrainingService.ts
@@ -529,7 +529,15 @@ class PAITrainingService implements TrainingService {

    private async statusCheckingLoop(): Promise<void> {
        while (!this.stopping) {
-            await this.updatePaiToken();
+            try{
+                await this.updatePaiToken();
+            }catch(error){
+                this.log.error(`${error}`);
+                //only throw error when initlize paiToken first time
+                if(!this.paiToken) {
+                    throw new Error(error);
+                }
+            }
            await this.paiJobCollector.retrieveTrialStatus(this.paiToken, this.paiClusterConfig);
            const restServer: PAIJobRestServer = component.get(PAIJobRestServer);
            if (restServer.getErrorMessage) {