quick fix unknow trial report to resume experiment (#3096)

nni/runtime/msg_dispatcher.py

@@ -39,7 +39,7 @@ def _sort_history(history):
 # Tuner global variables
 _next_parameter_id = 0
 _trial_params = {}
-'''key: trial job ID; value: parameters'''
+'''key: parameter ID; value: parameters'''
 _customized_parameter_ids = set()
 
 
@@ -182,8 +182,11 @@ class MsgDispatcher(MsgDispatcherBase):
             customized = True
         else:
             customized = False
+        if id_ in _trial_params:
             self.tuner.receive_trial_result(id_, _trial_params[id_], value, customized=customized,
                                             trial_job_id=data.get('trial_job_id'))
+        else:
+            _logger.warning('Find unknown job parameter id %s, maybe something goes wrong.', _trial_params[id_])
 
     def _handle_intermediate_metric_data(self, data):
         """Call assessor to process intermediate results

ts/nni_manager/core/nnimanager.ts

@@ -678,11 +678,15 @@ class NNIManager implements Manager {
 
     private async onTrialJobMetrics(metric: TrialJobMetric): Promise<void> {
         this.log.debug(`NNIManager received trial job metrics: ${metric}`);
+        if (this.trialJobs.has(metric.id)){
             await this.dataStore.storeMetricData(metric.id, metric.data);
             if (this.dispatcher === undefined) {
                 throw new Error('Error: tuner has not been setup');
             }
             this.dispatcher.sendCommand(REPORT_METRIC_DATA, metric.data);
+        } else {
+            this.log.warning(`NNIManager received non-existent trial job metrics: ${metric}`);
+        }
     }
 
     private requestTrialJobs(jobNum: number): void {