Make pylint happy (#1649)

Update python sdk and nni_annotation to pass pylint rules

src/sdk/pynni/nni/protocol.py

@@ -21,7 +21,6 @@
 import logging
 import threading
 from enum import Enum
-from .common import multi_thread_enabled
 
 
 class CommandType(Enum):
@@ -49,7 +48,6 @@ try:
     _out_file = open(4, 'wb')
 except OSError:
     _msg = 'IPC pipeline not exists, maybe you are importing tuner/assessor from trial code?'
-    import logging
     logging.getLogger(__name__).warning(_msg)
 
 
@@ -64,7 +62,7 @@ def send(command, data):
         data = data.encode('utf8')
         assert len(data) < 1000000, 'Command too long'
         msg = b'%b%06d%b' % (command.value, len(data), data)
-        logging.getLogger(__name__).debug('Sending command, data: [%s]' % msg)
+        logging.getLogger(__name__).debug('Sending command, data: [%s]', msg)
         _out_file.write(msg)
         _out_file.flush()
     finally:
@@ -76,7 +74,7 @@ def receive():
     Returns a tuple of command (CommandType) and payload (str)
     """
     header = _in_file.read(8)
-    logging.getLogger(__name__).debug('Received command, header: [%s]' % header)
+    logging.getLogger(__name__).debug('Received command, header: [%s]', header)
     if header is None or len(header) < 8:
         # Pipe EOF encountered
         logging.getLogger(__name__).debug('Pipe EOF encountered')
@@ -85,5 +83,5 @@ def receive():
     data = _in_file.read(length)
     command = CommandType(header[:2])
     data = data.decode('utf8')
-    logging.getLogger(__name__).debug('Received command, data: [%s]' % data)
+    logging.getLogger(__name__).debug('Received command, data: [%s]', data)
     return command, data

src/sdk/pynni/nni/recoverable.py

@@ -21,6 +21,7 @@
 import os
 
 class Recoverable:
+
     def load_checkpoint(self):
         pass
 
@@ -31,4 +32,4 @@ class Recoverable:
         ckp_path = os.getenv('NNI_CHECKPOINT_DIRECTORY')
         if ckp_path is not None and os.path.isdir(ckp_path):
             return ckp_path
-        return None
\ No newline at end of file
+        return None

src/sdk/pynni/nni/smac_tuner/smac_tuner.py

@@ -25,15 +25,18 @@ import logging
 import sys
 
 import numpy as np
-from ConfigSpaceNNI import Configuration
-from nni.tuner import Tuner
-from nni.utils import OptimizeMode, extract_scalar_reward
+
 from smac.facade.epils_facade import EPILS
 from smac.facade.roar_facade import ROAR
 from smac.facade.smac_facade import SMAC
 from smac.scenario.scenario import Scenario
 from smac.utils.io.cmd_reader import CMDReader
 
+from ConfigSpaceNNI import Configuration
+
+from nni.tuner import Tuner
+from nni.utils import OptimizeMode, extract_scalar_reward
+
 from .convert_ss_to_scenario import generate_scenario
 
 
@@ -72,11 +75,9 @@ class SMACTuner(Tuner):
 
         root_logger = logging.getLogger()
         root_logger.setLevel(args.verbose_level)
-        logger_handler = logging.StreamHandler(
-                stream=sys.stdout)
+        logger_handler = logging.StreamHandler(stream=sys.stdout)
         if root_logger.level >= logging.INFO:
-            formatter = logging.Formatter(
-                "%(levelname)s:\t%(message)s")
+            formatter = logging.Formatter("%(levelname)s:\t%(message)s")
         else:
             formatter = logging.Formatter(
                 "%(asctime)s:%(levelname)s:%(name)s:%(message)s",

src/sdk/pynni/nni/smartparam.py

@@ -43,8 +43,6 @@ __all__ = [
 ]
 
 
-# pylint: disable=unused-argument
-
 if trial_env_vars.NNI_PLATFORM is None:
     def choice(*options, name=None):
         return param_exp.choice(options, np.random.RandomState())
@@ -150,42 +148,16 @@ else:
         optional_input_size: number of candidate inputs to be chosen
         tf: tensorflow module
         '''
+        args = (mutable_id, mutable_layer_id, funcs, funcs_args, fixed_inputs, optional_inputs, optional_input_size)
         if mode == 'classic_mode':
-            return classic_mode(mutable_id,
-                            mutable_layer_id,
-                            funcs,
-                            funcs_args,
-                            fixed_inputs,
-                            optional_inputs,
-                            optional_input_size)
+            return classic_mode(*args)
         assert tf is not None, 'Internal Error: Tensorflow should not be None in modes other than classic_mode'
         if mode == 'enas_mode':
-            return enas_mode(mutable_id,
-                            mutable_layer_id,
-                            funcs,
-                            funcs_args,
-                            fixed_inputs,
-                            optional_inputs,
-                            optional_input_size,
-                            tf)
+            return enas_mode(*args, tf)
         if mode == 'oneshot_mode':
-            return oneshot_mode(mutable_id,
-                            mutable_layer_id,
-                            funcs,
-                            funcs_args,
-                            fixed_inputs,
-                            optional_inputs,
-                            optional_input_size,
-                            tf)
+            return oneshot_mode(*args, tf)
         if mode == 'darts_mode':
-            return darts_mode(mutable_id,
-                            mutable_layer_id,
-                            funcs,
-                            funcs_args,
-                            fixed_inputs,
-                            optional_inputs,
-                            optional_input_size,
-                            tf)
+            return darts_mode(*args, tf)
         raise RuntimeError('Unrecognized mode: %s' % mode)
 
     def _get_param(key):

src/sdk/pynni/nni/trial.py

@@ -96,7 +96,7 @@ def report_final_result(metric):
         'parameter_id': _params['parameter_id'],
         'trial_job_id': trial_env_vars.NNI_TRIAL_JOB_ID,
         'type': 'FINAL',
-        'sequence': 0,  # TODO: may be unnecessary
+        'sequence': 0,
         'value': metric
     })
     platform.send_metric(metric)

src/sdk/pynni/nni/tuner.py

@@ -27,7 +27,6 @@ _logger = logging.getLogger(__name__)
 
 
 class Tuner(Recoverable):
-    # pylint: disable=no-self-use,unused-argument
 
     def generate_parameters(self, parameter_id, **kwargs):
         """Returns a set of trial (hyper-)parameters, as a serializable object.
@@ -47,7 +46,7 @@ class Tuner(Recoverable):
         result = []
         for parameter_id in parameter_id_list:
             try:
-                _logger.debug("generating param for {}".format(parameter_id))
+                _logger.debug("generating param for %s", parameter_id)
                 res = self.generate_parameters(parameter_id, **kwargs)
             except nni.NoMoreTrialError:
                 return result
@@ -67,10 +66,12 @@ class Tuner(Recoverable):
         raise NotImplementedError('Tuner: receive_trial_result not implemented')
 
     def accept_customized_trials(self, accept=True):
-        """Enable or disable receiving results of user-added hyper-parameters. 
+        """Enable or disable receiving results of user-added hyper-parameters.
         By default `receive_trial_result()` will only receive results of algorithm-generated hyper-parameters.
         If tuners want to receive those of customized parameters as well, they can call this function in `__init__()`.
         """
+        # pylint: disable=attribute-defined-outside-init
+        # FIXME: because tuner is designed as interface, this API should not be here
         self._accept_customized = accept
 
     def trial_end(self, parameter_id, success, **kwargs):
@@ -78,7 +79,6 @@ class Tuner(Recoverable):
         parameter_id: int
         success: True if the trial successfully completed; False if failed or terminated
         """
-        pass
 
     def update_search_space(self, search_space):
         """Update the search space of tuner. Must override.
@@ -91,20 +91,19 @@ class Tuner(Recoverable):
         path: checkpoint directory for tuner
         """
         checkpoin_path = self.get_checkpoint_path()
-        _logger.info('Load checkpoint ignored by tuner, checkpoint path: %s' % checkpoin_path)
+        _logger.info('Load checkpoint ignored by tuner, checkpoint path: %s', checkpoin_path)
 
     def save_checkpoint(self):
         """Save the checkpoint of tuner.
         path: checkpoint directory for tuner
         """
         checkpoin_path = self.get_checkpoint_path()
-        _logger.info('Save checkpoint ignored by tuner, checkpoint path: %s' % checkpoin_path)
+        _logger.info('Save checkpoint ignored by tuner, checkpoint path: %s', checkpoin_path)
 
     def import_data(self, data):
         """Import additional data for tuning
         data: a list of dictionarys, each of which has at least two keys, 'parameter' and 'value'
         """
-        pass
 
     def _on_exit(self):
         pass

src/sdk/pynni/nni/utils.py

@@ -84,12 +84,13 @@ def extract_scalar_reward(value, scalar_key='default'):
         Incorrect final result: the final result should be float/int,
         or a dict which has a key named "default" whose value is float/int.
     """
-    if isinstance(value, float) or isinstance(value, int):
+    if isinstance(value, (float, int)):
         reward = value
     elif isinstance(value, dict) and scalar_key in value and isinstance(value[scalar_key], (float, int)):
         reward = value[scalar_key]
     else:
-        raise RuntimeError('Incorrect final result: the final result should be float/int, or a dict which has a key named "default" whose value is float/int.')
+        raise RuntimeError('Incorrect final result: the final result should be float/int, ' \
+            'or a dict which has a key named "default" whose value is float/int.')
     return reward
 
 
@@ -101,8 +102,7 @@ def convert_dict2tuple(value):
         for _keys in value:
             value[_keys] = convert_dict2tuple(value[_keys])
         return tuple(sorted(value.items()))
-    else:
-        return value
+    return value
 
 
 def init_dispatcher_logger():

src/sdk/pynni/pylintrc

+# This pylintrc file is a little more strick than the one in root of code directory
+# SDK source MUST pass lint rules in top level directory, and SHOULD pass rules here
+
+[SETTINGS]
+
+max-line-length=140
+
+disable =
+    missing-docstring,
+    invalid-name,                   # C0103
+    no-member,                      # E1101: sometimes pylint cannot detect members correctly due to a bug
+    c-extension-no-member,          # I1101
+    no-self-use,                    # R0201: many functions in this SDK are designed for override
+    duplicate-code,                 # R0801
+    too-many-instance-attributes,   # R0902
+    too-few-public-methods,         # R0903
+    too-many-public-methods,        # R0904
+    too-many-return-statements,     # R0911
+    too-many-branches,              # R0912
+    too-many-arguments,             # R0913
+    too-many-locals,                # R0914
+    too-many-statements,            # R0915
+    too-many-nested-blocks,         # R1702
+    no-else-return,                 # R1705
+    chained-comparison,             # R1716
+    no-else-raise,                  # R1720
+    protected-access,               # W0212: underscore variables may be protected by whole SDK instead of single module
+    arguments-differ,               # W0221: pylint cannot handle *args and **kwargs
+    super-init-not-called,          # W0231: some interface classes do not expect users to call init
+    useless-super-delegation,       # W0235: derived init may have different docstring
+    global-statement,               # W0603: globals are useful to hide SDK internal states from user
+    unused-argument,                # W0613: many functions in this SDK are designed for override
+    broad-except,                   # W0703: the SDK commonly catch exceptions to report error
+    fixme,                          # W0511
+
+ignore-patterns=test.*.py
+
+# List of members which are set dynamically and missed by pylint inference
+generated-members=numpy.*,torch.*

src/webui/src/App.tsx

@@ -33,7 +33,7 @@ class App extends React.Component<{}, AppState> {
     }
 
     changeInterval = (interval: number) => {
-        this.setState({ interval: interval });
+        this.setState({ interval });
         if (this.timerId === null && interval !== 0) {
             window.setTimeout(this.refresh);
         } else if (this.timerId !== null && interval === 0) {

tools/nni_annotation/__init__.py

@@ -154,4 +154,3 @@ def _generate_specific_file(src_path, dst_path, exp_id, trial_id, module):
                 raise RuntimeError(src_path + ' ' + '\n'.join(str(arg) for arg in exc.args))
             else:
                 raise RuntimeError('Failed to expand annotations for %s: %r' % (src_path, exc))
-

tools/nni_annotation/search_space_generator.py

@@ -20,9 +20,10 @@
 
 
 import ast
-import astor
 import numbers
 
+import astor
+
 # pylint: disable=unidiomatic-typecheck