Fix final metrics collection when metrics data is not at beginning of line (#1293)

* Fix final metrics (#1289)

* Fix final metrics with trial keeper

test/metrics_test/trial.py

@@ -2,9 +2,14 @@ import time
 import nni
 
 if __name__ == '__main__':
-    hyper_params = nni.get_next_parameter()
-
+    nni.get_next_parameter()
     for i in range(10):
+        if i % 2 == 0:
+            print('report intermediate result without end of line.', end='')
+        else:
+            print('report intermediate result.')
         nni.report_intermediate_result(0.1*(i+1))
         time.sleep(2)
+    print('test final metrics not at line start.', end='')
     nni.report_final_result(1.0)
+    print('done')

test/pipelines-it-pai.yml

@@ -76,4 +76,5 @@ jobs:
       --nni_docker_image $TEST_IMG --data_dir $(data_dir) --output_dir $(output_dir) --nni_manager_ip $(nni_manager_ip)
 
       PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts pai --exclude multi_phase_batch,multi_phase_grid
+      PATH=$HOME/.local/bin:$PATH python3 metrics_test.py
     displayName: 'integration test'

test/pipelines-it-remote.yml

@@ -53,6 +53,7 @@ jobs:
       --remote_port $(cat port) --remote_pwd $(docker_pwd) --nni_manager_ip $(nni_manager_ip)
       cat training_service.yml
       PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts remote --exclude cifar10,multi_phase_batch,multi_phase_grid
+      PATH=$HOME/.local/bin:$PATH python3 metrics_test.py
     displayName: 'integration test'
   - task: SSH@0
     inputs:

tools/nni_trial_tool/log_utils.py

@@ -134,7 +134,7 @@ class PipeLogReader(threading.Thread):
         self._is_read_completed = False
         self.process_exit = False
         self.log_collection = log_collection
-        self.log_pattern = re.compile(r'^NNISDK_MEb\'.*\'$')
+        self.log_pattern = re.compile(r'NNISDK_MEb\'.*\'$')
 
         def _populateQueue(stream, queue):
             '''
@@ -172,10 +172,13 @@ class PipeLogReader(threading.Thread):
         for line in iter(self.pipeReader.readline, ''):
             self.orig_stdout.write(line.rstrip() + '\n')
             self.orig_stdout.flush()
+
             if self.log_collection == 'none':
-                # If not match metrics, do not put the line into queue
-                if not self.log_pattern.match(line):
-                    continue
+                search_result = self.log_pattern.search(line)
+                if search_result:
+                    metrics = search_result.group(0)
+                    self.queue.put(metrics+'\n')
+            else:
                 self.queue.put(line)
 
         self.pipeReader.close()