move reward extraction logic to tuner (#274)

* add pycharm project files to .gitignore list

* update pylintrc to conform vscode settings

* fix RemoteMachineMode for wrong trainingServicePlatform

* add python cache files to gitignore list

* move extract scalar reward logic from dispatcher to tuner

* update tuner code corresponding to last commit

* update doc for receive_trial_result api change

* add numpy to package whitelist of pylint

* distinguish param value from return reward for tuner.extract_scalar_reward

* update pylintrc

* add comments to dispatcher.handle_report_metric_data

* refactor extract reward from dict by tuner

.gitignore

@@ -62,3 +62,8 @@ typings/
 
 # Pycharm Project files
 .idea
+
+# Python cache files
+__pycache__
+build
+*.egg-info
\ No newline at end of file

docs/howto_2_CustomizedTuner.md

@@ -27,12 +27,12 @@ class CustomizedTuner(Tuner):
     def __init__(self, ...):
         ...
     
-    def receive_trial_result(self, parameter_id, parameters, reward):
+    def receive_trial_result(self, parameter_id, parameters, value):
     '''
     Record an observation of the objective function and Train
     parameter_id: int
     parameters: object created by 'generate_parameters()'
-    reward: object reported by trial
+    value: final metrics of the trial, including reward
     '''
     # your code implements here.
     ...
@@ -46,7 +46,7 @@ class CustomizedTuner(Tuner):
     return your_parameters
     ...
 ```
-```receive_trial_result``` will receive ```the parameter_id, parameters, reward``` as parameters input. Also, Tuner will receive the ```reward``` object are exactly same reward that Trial send.
+```receive_trial_result``` will receive ```the parameter_id, parameters, value``` as parameters input. Also, Tuner will receive the ```value``` object are exactly same value that Trial send.
 
 The ```your_parameters``` return from ```generate_parameters``` function, will be package as json object by NNI SDK. NNI SDK will unpack json object so the Trial will receive the exact same ```your_parameters``` from Tuner.
 
@@ -65,7 +65,7 @@ If the you implement the ```generate_parameters``` like this:
 ```
 parameter_id = 82347
 parameters = {"dropout": 0.3, "learning_rate": 0.4}
-reward = 0.93
+value = 0.93
 ```
 
 **Note that** if you want to access a file (e.g., ```data.txt```) in the directory of your own tuner, you cannot use ```open('data.txt', 'r')```. Instead, you should use the following:

examples/tuners/ga_customer_tuner/customer_tuner.py

@@ -108,13 +108,14 @@ class CustomerTuner(Tuner):
         return temp
 
 
-    def receive_trial_result(self, parameter_id, parameters, reward):
+    def receive_trial_result(self, parameter_id, parameters, value):
         '''
         Record an observation of the objective function
         parameter_id : int
         parameters : dict of parameters
-        reward : reward of one trial
+        value: final metrics of the trial, including reward
         '''
+        reward = self.extract_scalar_reward(value)
         if self.optimize_mode is OptimizeMode.Minimize:
             reward = -reward
 
@@ -131,7 +132,7 @@ class CustomerTuner(Tuner):
 
 if __name__ =='__main__':
     tuner = CustomerTuner(OptimizeMode.Maximize)
-    config = tuner.generate_parameter(0)
+    config = tuner.generate_parameters(0)
     with open('./data.json', 'w') as outfile:
         json.dump(config, outfile)
     tuner.receive_trial_result(0, config, 0.99)

pylintrc

@@ -22,8 +22,9 @@ enable=F,
        duplicate-key,
        unnecessary-semicolon,
        global-variable-not-assigned,
-       unused-variable,
        binary-op-exception,
        bad-format-string,
        anomalous-backslash-in-string,
        bad-open-mode
+
+extension-pkg-whitelist=numpy
\ No newline at end of file

src/nni_manager/core/test/dummy_tuner.py

@@ -25,10 +25,10 @@ class DummyTuner(Tuner):
     def generate_multiple_parameters(self, parameter_id_list):
         return ['unit-test-param1', 'unit-test-param2']
 
-    def receive_trial_result(self, parameter_id, parameters, reward):
+    def receive_trial_result(self, parameter_id, parameters, value):
         pass
 
-    def receive_customized_trial_result(self, parameter_id, parameters, reward):
+    def receive_customized_trial_result(self, parameter_id, parameters, value):
         pass
 
     def update_search_space(self, search_space):

src/sdk/pynni/nni/batch_tuner/batch_tuner.py

@@ -77,5 +77,5 @@ class BatchTuner(Tuner):
             raise nni.NoMoreTrialError('no more parameters now.')
         return self.values[self.count]
 
-    def receive_trial_result(self, parameter_id, parameters, reward):
+    def receive_trial_result(self, parameter_id, parameters, value):
         pass
\ No newline at end of file

src/sdk/pynni/nni/evolution_tuner/evolution_tuner.py

@@ -234,12 +234,13 @@ class EvolutionTuner(Tuner):
         config = _split_index(total_config)
         return config
 
-    def receive_trial_result(self, parameter_id, parameters, reward):
+    def receive_trial_result(self, parameter_id, parameters, value):
         '''
         Record an observation of the objective function
         parameters: dict of parameters
-        reward: reward of one trial
+        value: final metrics of the trial, including reward
         '''
+        reward = self.extract_scalar_reward(value)
         if parameter_id not in self.total_data:
             raise RuntimeError('Received parameter_id not in total_data.')
         # restore the paramsters contains "_index"

src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py

@@ -206,13 +206,14 @@ class HyperoptTuner(Tuner):
         params = _split_index(total_params)
         return params
 
-    def receive_trial_result(self, parameter_id, parameters, reward):
+    def receive_trial_result(self, parameter_id, parameters, value):
         '''
         Record an observation of the objective function
         parameter_id : int
         parameters : dict of parameters
-        reward : reward of one trial
+        value: final metrics of the trial, including reward
         '''
+        reward = self.extract_scalar_reward(value)
         # restore the paramsters contains '_index'
         if parameter_id not in self.total_data:
             raise RuntimeError('Received parameter_id not in total_data.')

src/sdk/pynni/nni/msg_dispatcher.py

@@ -110,21 +110,15 @@ class MsgDispatcher(MsgDispatcherBase):
         return True
 
     def handle_report_metric_data(self, data):
+        """
+        :param data: a dict received from nni_manager, which contains:
+                    - 'parameter_id': id of the trial
+                    - 'value': metric value reported by nni.report_final_result()
+                    - 'type': report type, support {'FINAL', 'PERIODICAL'}
+        """
         if data['type'] == 'FINAL':
-            value = None
             id_ = data['parameter_id']
-            
-            if isinstance(data['value'], float) or isinstance(data['value'], int):
             value = data['value']
-            elif isinstance(data['value'], dict) and 'default' in data['value']:
-                value = data['value']['default']
-                if isinstance(value, float) or isinstance(value, int):
-                    pass
-                else:
-                    raise RuntimeError('Incorrect final result: the final result should be float/int, or a dict which has a key named "default" whose value is float/int.')
-            else:
-                raise RuntimeError('Incorrect final result: the final result should be float/int, or a dict which has a key named "default" whose value is float/int.') 
-            
             if id_ in _customized_parameter_ids:
                 self.tuner.receive_customized_trial_result(id_, _trial_params[id_], value)
             else:

src/sdk/pynni/nni/multi_phase/multi_phase_tuner.py

@@ -44,19 +44,19 @@ class MultiPhaseTuner(Recoverable):
         """
         return [self.generate_parameters(parameter_id) for parameter_id in parameter_id_list]
 
-    def receive_trial_result(self, parameter_id, parameters, reward, trial_job_id):
+    def receive_trial_result(self, parameter_id, parameters, value, trial_job_id):
         """Invoked when a trial reports its final result. Must override.
         parameter_id: int
         parameters: object created by 'generate_parameters()'
-        reward: object reported by trial
+        value: object reported by trial
         """
         raise NotImplementedError('Tuner: receive_trial_result not implemented')
 
-    def receive_customized_trial_result(self, parameter_id, parameters, reward, trial_job_id):
+    def receive_customized_trial_result(self, parameter_id, parameters, value, trial_job_id):
         """Invoked when a trial added by WebUI reports its final result. Do nothing by default.
         parameter_id: int
         parameters: object created by user
-        reward: object reported by trial
+        value: object reported by trial
         """
         _logger.info('Customized trial job %s ignored by tuner', parameter_id)
 

src/sdk/pynni/nni/smac_tuner/smac_tuner.py

@@ -134,10 +134,11 @@ class SMACTuner(Tuner):
         else:
             self.logger.warning('update search space is not supported.')
 
-    def receive_trial_result(self, parameter_id, parameters, reward):
+    def receive_trial_result(self, parameter_id, parameters, value):
         '''
         receive_trial_result
         '''
+        reward = self.extract_scalar_reward(value)
         if self.optimize_mode is OptimizeMode.Maximize:
             reward = -reward
 

src/sdk/pynni/nni/tuner.py

@@ -52,7 +52,7 @@ class Tuner(Recoverable):
             result.append(res)
         return result
 
-    def receive_trial_result(self, parameter_id, parameters, reward):
+    def receive_trial_result(self, parameter_id, parameters, value):
         """Invoked when a trial reports its final result. Must override.
         parameter_id: int
         parameters: object created by 'generate_parameters()'
@@ -60,11 +60,11 @@ class Tuner(Recoverable):
         """
         raise NotImplementedError('Tuner: receive_trial_result not implemented')
 
-    def receive_customized_trial_result(self, parameter_id, parameters, reward):
+    def receive_customized_trial_result(self, parameter_id, parameters, value):
         """Invoked when a trial added by WebUI reports its final result. Do nothing by default.
         parameter_id: int
         parameters: object created by user
-        reward: object reported by trial
+        value: object reported by trial
         """
         _logger.info('Customized trial job %s ignored by tuner', parameter_id)
 
@@ -93,3 +93,12 @@ class Tuner(Recoverable):
 
     def _on_error(self):
         pass
+
+    def extract_scalar_reward(self, value, scalar_key='default'):
+        if isinstance(value, float) or isinstance(value, int):
+            reward = value
+        elif isinstance(value, dict) and scalar_key in value and isinstance(value[scalar_key], (float, int)):
+            reward = value[scalar_key]
+        else:
+            raise RuntimeError('Incorrect final result: the final result for %s should be float/int, or a dict which has a key named "default" whose value is float/int.' % str(self.__class__)) 
+        return reward 
\ No newline at end of file

src/sdk/pynni/tests/test_multi_phase_tuner.py

@@ -35,10 +35,10 @@ class NaiveMultiPhaseTuner(MultiPhaseTuner):
         return generated_parameters
 
 
-    def receive_trial_result(self, parameter_id, parameters, reward, trial_job_id):
-        logging.getLogger(__name__).debug('receive_trial_result: {},{},{},{}'.format(parameter_id, parameters, reward, trial_job_id))
+    def receive_trial_result(self, parameter_id, parameters, value, trial_job_id):
+        logging.getLogger(__name__).debug('receive_trial_result: {},{},{},{}'.format(parameter_id, parameters, value, trial_job_id))
 
-    def receive_customized_trial_result(self, parameter_id, parameters, reward, trial_job_id):
+    def receive_customized_trial_result(self, parameter_id, parameters, value, trial_job_id):
         pass
 
     def update_search_space(self, search_space):

src/sdk/pynni/tests/test_tuner.py

@@ -44,10 +44,12 @@ class NaiveTuner(Tuner):
             'search_space': self.search_space
         }
 
-    def receive_trial_result(self, parameter_id, parameters, reward):
+    def receive_trial_result(self, parameter_id, parameters, value):
+        reward = self.extract_scalar_reward(value)
         self.trial_results.append((parameter_id, parameters['param'], reward, False))
 
-    def receive_customized_trial_result(self, parameter_id, parameters, reward):
+    def receive_customized_trial_result(self, parameter_id, parameters, value):
+        reward = self.extract_scalar_reward(value)
         self.trial_results.append((parameter_id, parameters['param'], reward, True))
 
     def update_search_space(self, search_space):

test/naive_test/naive_tuner.py

@@ -20,7 +20,8 @@ class NaiveTuner(Tuner):
         _logger.info('generate parameters: %s' % self.cur)
         return { 'x': self.cur }
 
-    def receive_trial_result(self, parameter_id, parameters, reward):
+    def receive_trial_result(self, parameter_id, parameters, value):
+        reward = self.extract_scalar_reward(value)
         _logger.info('receive trial result: %s, %s, %s' % (parameter_id, parameters, reward))
         _result.write('%d %d\n' % (parameters['x'], reward))
         _result.flush()