Catch dispatcher stream error event (#1850)

src/nni_manager/core/ipcInterface.ts

@@ -74,6 +74,8 @@ class IpcInterface {
         this.readBuffer = Buffer.alloc(0);
 
         this.incomingStream.on('data', (data: Buffer) => { this.receive(data); });
+        this.incomingStream.on('error', (error: Error) => { this.eventEmitter.emit('error', error); });
+        this.outgoingStream.on('error', (error: Error) => { this.eventEmitter.emit('error', error); });
     }
 
     /**
@@ -106,6 +108,10 @@ class IpcInterface {
         this.eventEmitter.on('command', listener);
     }
 
+    public onError(listener: (error: Error) => void): void {
+        this.eventEmitter.on('error', listener);
+    }
+
     /**
      * Deal with incoming data from process
      * Invoke listeners for each complete command received, save incomplete command to buffer

src/nni_manager/core/nnimanager.ts

@@ -652,6 +652,10 @@ class NNIManager implements Manager {
                 this.criticalError(NNIError.FromError(err, 'Tuner command event error: '));
             });
         });
+        this.dispatcher.onError((error: Error) => {
+            this.log.error(`Dispatcher error: ${error.message}`);
+            this.criticalError(new Error('Dispatcher stream error, tuner may have crashed.'));
+        });
     }
 
     private sendInitTunerCommands(): void {