Fix a race condidtion issue in trial_keeper for reading log from pipe (#578)

* Fix a race condidtion issue in trial_keeper for reading log from pipe

tools/nni_trial_tool/log_utils.py

@@ -129,6 +129,7 @@ class PipeLogReader(threading.Thread):
         self.pipeReader = os.fdopen(self.fdRead)
         self.orig_stdout = sys.__stdout__
         self._is_read_completed = False
+        self.process_exit = False
 
         def _populateQueue(stream, queue):
             '''
@@ -136,6 +137,7 @@ class PipeLogReader(threading.Thread):
             '''
             time.sleep(5)
             while True:
+                cur_process_exit = self.process_exit       
                 try:
                     line = self.queue.get(True, 5)
                     try:
@@ -145,6 +147,7 @@ class PipeLogReader(threading.Thread):
                     except Exception as e:
                         pass
                 except Exception as e:
+                    if cur_process_exit == True:     
                         self._is_read_completed = True
                         break
 
@@ -176,3 +179,7 @@ class PipeLogReader(threading.Thread):
         """Return if read is completed
         """
         return self._is_read_completed
+    
+    def set_process_exit(self):
+        self.process_exit = True
+        return self.process_exit
\ No newline at end of file

tools/nni_trial_tool/trial_keeper.py

@@ -65,7 +65,7 @@ def main_loop(args):
     while True:
         retCode = process.poll()
         # child worker process exits and all stdout data is read
-        if retCode is not None and log_pipe_stdout.is_read_completed == True:
+        if retCode is not None and log_pipe_stdout.set_process_exit() and log_pipe_stdout.is_read_completed == True:
             nni_log(LogType.Info, 'subprocess terminated. Exit code is {}. Quit'.format(retCode))
             if args.pai_hdfs_output_dir is not None:
                 # Copy local directory to hdfs for OpenPAI