[Enhancement] Test linting in CI and fix existing linting errors (#69)

* [Enhancement] Test linting in CI

* fix linting

.codespellrc

@@ -2,4 +2,4 @@
 skip = *.ipynb
 count =
 quiet-level = 3
-ignore-words-list = nd, ans, ques
+ignore-words-list = nd, ans, ques, rouge

.github/workflows/lint.yml

+name: lint
+
+on: [push, pull_request]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.10
+      - name: Install pre-commit hook
+        run: |
+          pip install pre-commit
+          pre-commit install
+      - name: Linting
+        run: pre-commit run --all-files

docs/zh_cn/conf.py

@@ -29,14 +29,12 @@ author = 'OpenCompass Authors'
 # The full version, including alpha/beta/rc tags
 # version_file = '../../opencompass/version.py'
 
-
 # def get_version():
 #     with open(version_file, 'r') as f:
 #         exec(compile(f.read(), version_file, 'exec'))
 #     return locals()['__version__']
 
-
-release = "0.5.0"
+release = '0.5.0'
 
 # -- General configuration ---------------------------------------------------
 
@@ -141,14 +139,16 @@ latex_elements = {
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (root_doc, 'opencompass.tex', 'OpenCompass Documentation', author, 'manual'),
+    (root_doc, 'opencompass.tex', 'OpenCompass Documentation', author,
+     'manual'),
 ]
 
 # -- Options for manual page output ------------------------------------------
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [(root_doc, 'opencompass', 'OpenCompass Documentation', [author], 1)]
+man_pages = [(root_doc, 'opencompass', 'OpenCompass Documentation', [author],
+              1)]
 
 # -- Options for Texinfo output ----------------------------------------------
 
@@ -156,8 +156,9 @@ man_pages = [(root_doc, 'opencompass', 'OpenCompass Documentation', [author], 1)
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (root_doc, 'opencompass', 'OpenCompass Documentation', author, 'OpenCompass Authors',
-     'AGI evaluation toolbox and benchmark.', 'Miscellaneous'),
+    (root_doc, 'opencompass', 'OpenCompass Documentation', author,
+     'OpenCompass Authors', 'AGI evaluation toolbox and benchmark.',
+     'Miscellaneous'),
 ]
 
 # -- Options for Epub output -------------------------------------------------

opencompass/datasets/ceval.py

@@ -12,19 +12,19 @@ class CEvalDataset(BaseDataset):
 
     @staticmethod
     def load(path: str, name: str):
-        dev_dataset = load_dataset(
-            'csv',
-            data_files=osp.join(path, 'dev', f'{name}_dev.csv'),
+        dev_dataset = load_dataset('csv',
+                                   data_files=osp.join(path, 'dev',
+                                                       f'{name}_dev.csv'),
                                    split='train')
-        val_dataset = load_dataset(
-            'csv',
-            data_files=osp.join(path, 'val', f'{name}_val.csv'),
+        val_dataset = load_dataset('csv',
+                                   data_files=osp.join(path, 'val',
+                                                       f'{name}_val.csv'),
                                    split='train')
         val_dataset = val_dataset.add_column('explanation',
                                              [''] * len(val_dataset))
-        test_dataset = load_dataset(
-            'csv',
-            data_files=osp.join(path, 'test', f'{name}_test.csv'),
+        test_dataset = load_dataset('csv',
+                                    data_files=osp.join(
+                                        path, 'test', f'{name}_test.csv'),
                                     split='train')
         test_dataset = test_dataset.add_column(
             'answer',

opencompass/datasets/flores.py

@@ -16,8 +16,9 @@ class FloresFirst100Dataset(BaseDataset):
             'dev':
             load_dataset(path='facebook/flores', name=name, split='dev'),
             'devtest':
-            load_dataset(
-                path='facebook/flores', name=name, split='devtest[:100]')
+            load_dataset(path='facebook/flores',
+                         name=name,
+                         split='devtest[:100]')
         })
 
 

opencompass/datasets/summedits.py

@@ -3,6 +3,7 @@ import json
 from datasets import Dataset
 
 from opencompass.registry import LOAD_DATASET
+
 from .base import BaseDataset
 
 

opencompass/models/huggingface.py

@@ -142,7 +142,8 @@ class HuggingFace(BaseModel):
         tokens = self.tokenizer.batch_encode_plus(inputs,
                                                   padding=True,
                                                   truncation=True,
-                                                  max_length=self.max_seq_len - max_out_len)
+                                                  max_length=self.max_seq_len -
+                                                  max_out_len)
         tokens = {
             k: torch.tensor(np.array(tokens[k]), device=self.model.device)
             for k in tokens if k in ['input_ids', 'attention_mask']
@@ -180,10 +181,10 @@ class HuggingFace(BaseModel):
 
         input_ids = self.tokenizer(inputs,
                                    truncation=True,
-                                   max_length=self.max_seq_len - max_out_len)['input_ids']
+                                   max_length=self.max_seq_len -
+                                   max_out_len)['input_ids']
         input_ids = torch.tensor(input_ids, device=self.model.device)
-        outputs = self.model.generate(input_ids,
-                                      max_new_tokens=max_out_len)
+        outputs = self.model.generate(input_ids, max_new_tokens=max_out_len)
 
         if not self.extract_pred_after_decode:
             outputs = outputs[:, input_ids.shape[1]:]

opencompass/openicl/__init__.py

-from .icl_dataset_reader import DatasetReader
-from .icl_evaluator import *
-from .icl_prompt_template import PromptTemplate
-from .icl_retriever import *
-from .icl_inferencer import *
+from .icl_dataset_reader import DatasetReader  # noqa
+from .icl_evaluator import *  # noqa
+from .icl_inferencer import *  # noqa
+from .icl_prompt_template import PromptTemplate  # noqa
+from .icl_retriever import *  # noqa

opencompass/openicl/icl_evaluator/__init__.py

-from .icl_aucroc_evaluator import AUCROCEvaluator
-from .icl_base_evaluator import BaseEvaluator
-from .icl_em_evaluator import EMEvaluator
+from .icl_aucroc_evaluator import AUCROCEvaluator  # noqa
+from .icl_base_evaluator import BaseEvaluator  # noqa
+from .icl_em_evaluator import EMEvaluator  # noqa
 from .icl_hf_evaluator import *  # noqa
-from .icl_toxic_evaluator import ToxicEvaluator
+from .icl_toxic_evaluator import ToxicEvaluator  # noqa

opencompass/openicl/icl_evaluator/icl_aucroc_evaluator.py

 from typing import List
+
 import numpy as np
 from sklearn.metrics import roc_auc_score
 

opencompass/openicl/icl_evaluator/icl_base_evaluator.py

 """Base Evaluator."""
-from typing import List
 
 
 class BaseEvaluator:
+
     def __init__(self) -> None:
         pass
 

opencompass/openicl/icl_evaluator/icl_hf_evaluator.py

 from typing import List
+
 import evaluate
 
 from opencompass.registry import ICL_EVALUATORS
@@ -54,9 +55,12 @@ class HuggingfaceEvaluator(BaseEvaluator):
             dict: calculated scores.
         """
         if len(predictions) != len(references):
-            return {'error': 'predictions and references have different '
+            return {
+                'error':
+                'predictions and references have different '
                 f'length. len(predictions): {len(predictions)}, '
-                f'len(references): {len(references)}'}
+                f'len(references): {len(references)}'
+            }
         metric = evaluate.load(self.metric)
         scores = metric.compute(**self._preprocess(predictions, references))
         return self._postprocess(scores)
@@ -103,7 +107,7 @@ class AccEvaluator(HuggingfaceEvaluator):
         Returns:
             dict: postprocessed scores.
         """
-        scores["accuracy"] *= 100
+        scores['accuracy'] *= 100
         return scores
 
 
@@ -150,7 +154,7 @@ class MccEvaluator(AccEvaluator):
         Returns:
             dict: postprocessed scores.
         """
-        scores["matthews_correlation"] *= 100
+        scores['matthews_correlation'] *= 100
         return scores
 
 

opencompass/openicl/icl_inferencer/__init__.py

-from .icl_base_inferencer import BaseInferencer
-from .icl_gen_inferencer import GenInferencer
-from .icl_ppl_inferencer import PPLInferencer
-from .icl_clp_inferencer import CLPInferencer
+from .icl_base_inferencer import BaseInferencer  # noqa
+from .icl_clp_inferencer import CLPInferencer  # noqa
+from .icl_gen_inferencer import GenInferencer  # noqa
+from .icl_ppl_inferencer import PPLInferencer  # noqa

opencompass/openicl/icl_inferencer/icl_clp_inferencer.py

@@ -98,8 +98,8 @@ class CLPInferencer(BaseInferencer):
         # 3. Generate in-context examples for testing inputs
         for idx in range(len(ice_idx_list)):
             ice.append(
-                retriever.generate_ice(
-                    ice_idx_list[idx], ice_template=ice_template))
+                retriever.generate_ice(ice_idx_list[idx],
+                                       ice_template=ice_template))
         output_handler.save_ice(ice)
 
         # 4. Collect prompts and calculate conditional log probs
@@ -165,8 +165,7 @@ class CLPInferencer(BaseInferencer):
                 choice_target_ids.append(prompt_token_num - 1)
 
             logger.info('Calculating conditional log probability for prompts.')
-            for idx in trange(
-                    0,
+            for idx in trange(0,
                               len(prompt_list),
                               self.batch_size,
                               disable=not self.is_main_process):

opencompass/openicl/icl_inferencer/icl_ppl_inferencer.py

@@ -25,7 +25,7 @@ class PPLInferencer(BaseInferencer):
         model (:obj:`BaseModel`, optional): The module to inference.
         max_seq_len (:obj:`int`): Maximum number of tokenized words allowed by
             the LM.
-        batch_size (:obj:`int`, optional): Batch size for the :obj:`DataLoader`.
+        batch_size (:obj:`int`, optional): Batch size for the :obj:`DataLoader`
         output_json_filepath (:obj:`str`, optional): File path for output
             `JSON` file.
         output_json_filename (:obj:`str`, optional): File name for output
@@ -126,7 +126,7 @@ class PPLInferencer(BaseInferencer):
                             label,
                             ice_template=ice_template,
                             prompt_template=prompt_template)
-                        prompt_token_num = self.model.get_token_len_from_template(
+                        prompt_token_num = self.model.get_token_len_from_template(  # noqa
                             prompt, mode='ppl')  # noqa
 
                 if normalizing_str is not None:

opencompass/openicl/icl_retriever/__init__.py

-from .icl_base_retriever import BaseRetriever
-from .icl_bm25_retriever import BM25Retriever
-from .icl_dpp_retriever import DPPRetriever
-from .icl_fix_k_retriever import FixKRetriever
-from .icl_mdl_retriever import MDLRetriever
-from .icl_random_retriever import RandomRetriever
-from .icl_topk_retriever import TopkRetriever
-from .icl_votek_retriever import VotekRetriever
-from .icl_zero_retriever import ZeroRetriever
+from .icl_base_retriever import BaseRetriever  # noqa
+from .icl_bm25_retriever import BM25Retriever  # noqa
+from .icl_dpp_retriever import DPPRetriever  # noqa
+from .icl_fix_k_retriever import FixKRetriever  # noqa
+from .icl_mdl_retriever import MDLRetriever  # noqa
+from .icl_random_retriever import RandomRetriever  # noqa
+from .icl_topk_retriever import TopkRetriever  # noqa
+from .icl_votek_retriever import VotekRetriever  # noqa
+from .icl_zero_retriever import ZeroRetriever  # noqa