[Sync] Fix cmnli, fix vicuna meta template, fix longbench postprocess and other minor fixes (#625)

configs/datasets/CLUE_cmnli/CLUE_cmnli_ppl_98dd6e.py

@@ -2,7 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import PPLInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
-from opencompass.datasets import HFDataset
+from opencompass.datasets import cmnliDataset
 
 cmnli_reader_cfg = dict(
     input_columns=['sentence1', 'sentence2'],
@@ -25,11 +25,9 @@ cmnli_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
 
 cmnli_datasets = [
     dict(
-        type=HFDataset,
-        abbr='cmnli',
-        path='json',
-        split='train',
-        data_files='./data/CLUE/cmnli/cmnli_public/dev.json',
+        abbr="cmnli",
+        type=cmnliDataset,
+        path='./data/CLUE/cmnli/cmnli_public/dev.json',
         reader_cfg=cmnli_reader_cfg,
         infer_cfg=cmnli_infer_cfg,
         eval_cfg=cmnli_eval_cfg)

configs/datasets/CLUE_cmnli/CLUE_cmnli_ppl_ef69e7.py

@@ -2,7 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import PPLInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
-from opencompass.datasets import HFDataset
+from opencompass.datasets import cmnliDataset
 
 cmnli_reader_cfg = dict(
     input_columns=['sentence1', 'sentence2'],
@@ -41,11 +41,9 @@ cmnli_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
 
 cmnli_datasets = [
     dict(
-        type=HFDataset,
-        abbr='cmnli',
-        path='json',
-        split='train',
-        data_files='./data/CLUE/cmnli/cmnli_public/dev.json',
+        abbr="cmnli",
+        type=cmnliDataset,
+        path='./data/CLUE/cmnli/cmnli_public/dev.json',
         reader_cfg=cmnli_reader_cfg,
         infer_cfg=cmnli_infer_cfg,
         eval_cfg=cmnli_eval_cfg)

configs/datasets/CLUE_cmnli/CLUE_cmnli_ppl_fdc6de.py

@@ -2,7 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import PPLInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
-from opencompass.datasets import HFDataset
+from opencompass.datasets import cmnliDataset
 
 cmnli_reader_cfg = dict(
     input_columns=['sentence1', 'sentence2'],
@@ -45,11 +45,9 @@ cmnli_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
 
 cmnli_datasets = [
     dict(
-        type=HFDataset,
-        abbr='cmnli',
-        path='json',
-        split='train',
-        data_files='./data/CLUE/cmnli/cmnli_public/dev.json',
+        abbr="cmnli",
+        type=cmnliDataset,
+        path='./data/CLUE/cmnli/cmnli_public/dev.json',
         reader_cfg=cmnli_reader_cfg,
         infer_cfg=cmnli_infer_cfg,
         eval_cfg=cmnli_eval_cfg)

configs/datasets/longbench/longbenchlsht/longbench_lsht_gen_e8a339.py

 from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
-from opencompass.datasets import LongBenchClassificationEvaluator, LongBenchlshtDataset
+from opencompass.datasets import LongBenchClassificationEvaluator, LongBenchlshtDataset, lsht_postprocess
 
 LongBench_lsht_reader_cfg = dict(
     input_columns=['context', 'input'],
@@ -23,7 +23,8 @@ LongBench_lsht_infer_cfg = dict(
 
 LongBench_lsht_eval_cfg = dict(
     evaluator=dict(type=LongBenchClassificationEvaluator),
-    pred_role='BOT'
+    pred_role='BOT',
+    pred_postprocessor=dict(type=lsht_postprocess),
 )
 
 LongBench_lsht_datasets = [

configs/datasets/longbench/longbenchsamsum/longbench_samsum_gen_f4416d.py

 from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
-from opencompass.datasets import LongBenchRougeEvaluator, LongBenchsamsumDataset
+from opencompass.datasets import LongBenchRougeEvaluator, LongBenchsamsumDataset, samsum_postprocess
 
 LongBench_samsum_reader_cfg = dict(
     input_columns=['context', 'input'],
@@ -23,7 +23,8 @@ LongBench_samsum_infer_cfg = dict(
 
 LongBench_samsum_eval_cfg = dict(
     evaluator=dict(type=LongBenchRougeEvaluator),
-    pred_role='BOT'
+    pred_role='BOT',
+    pred_postprocessor=dict(type=samsum_postprocess),
 )
 
 LongBench_samsum_datasets = [

configs/datasets/longbench/longbenchtrec/longbench_trec_gen_824187.py

 from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
-from opencompass.datasets import LongBenchClassificationEvaluator, LongBenchtrecDataset
+from opencompass.datasets import LongBenchClassificationEvaluator, LongBenchtrecDataset, trec_postprocess
 
 LongBench_trec_reader_cfg = dict(
     input_columns=['context', 'input'],
@@ -23,7 +23,8 @@ LongBench_trec_infer_cfg = dict(
 
 LongBench_trec_eval_cfg = dict(
     evaluator=dict(type=LongBenchClassificationEvaluator),
-    pred_role='BOT'
+    pred_role='BOT',
+    pred_postprocessor=dict(type=trec_postprocess),
 )
 
 LongBench_trec_datasets = [

configs/datasets/longbench/longbenchtriviaqa/longbench_triviaqa_gen_d30cb9.py

 from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
-from opencompass.datasets import LongBenchF1Evaluator, LongBenchtriviaqaDataset
+from opencompass.datasets import LongBenchF1Evaluator, LongBenchtriviaqaDataset, triviaqa_postprocess
 
 LongBench_triviaqa_reader_cfg = dict(
     input_columns=['context', 'input'],
@@ -23,7 +23,8 @@ LongBench_triviaqa_infer_cfg = dict(
 
 LongBench_triviaqa_eval_cfg = dict(
     evaluator=dict(type=LongBenchF1Evaluator),
-    pred_role='BOT'
+    pred_role='BOT',
+    pred_postprocessor=dict(type=triviaqa_postprocess),
 )
 
 LongBench_triviaqa_datasets = [

configs/models/vicuna/hf_vicuna_13b_v13.py

@@ -17,6 +17,7 @@ models = [
         batch_size=8,
         model_kwargs=dict(device_map='auto'),
         batch_padding=False, # if false, inference with for-loop without batch padding
+        use_fastchat_template=True,
         run_cfg=dict(num_gpus=2, num_procs=1)
     )
 ]

configs/models/vicuna/hf_vicuna_13b_v15.py

@@ -17,6 +17,7 @@ models = [
         batch_size=8,
         model_kwargs=dict(device_map='auto'),
         batch_padding=False, # if false, inference with for-loop without batch padding
+        use_fastchat_template=True,
         run_cfg=dict(num_gpus=1, num_procs=1)
     )
 ]

configs/models/vicuna/hf_vicuna_13b_v15_16k.py

@@ -17,6 +17,7 @@ models = [
         batch_size=8,
         model_kwargs=dict(device_map='auto'),
         batch_padding=False, # if false, inference with for-loop without batch padding
+        use_fastchat_template=True,
         run_cfg=dict(num_gpus=2, num_procs=1)
     )
 ]

configs/models/vicuna/hf_vicuna_33b_v13.py

@@ -17,6 +17,7 @@ models = [
         batch_size=8,
         model_kwargs=dict(device_map='auto'),
         batch_padding=False, # if false, inference with for-loop without batch padding
+        use_fastchat_template=True,
         run_cfg=dict(num_gpus=4, num_procs=1)
     )
 ]

configs/models/vicuna/hf_vicuna_7b_v13.py

@@ -17,6 +17,7 @@ models = [
         batch_size=8,
         model_kwargs=dict(device_map='auto'),
         batch_padding=False, # if false, inference with for-loop without batch padding
+        use_fastchat_template=True,
         run_cfg=dict(num_gpus=1, num_procs=1)
     )
 ]

configs/models/vicuna/hf_vicuna_7b_v15.py

@@ -17,6 +17,7 @@ models = [
         batch_size=8,
         model_kwargs=dict(device_map='auto'),
         batch_padding=False, # if false, inference with for-loop without batch padding
+        use_fastchat_template=True,
         run_cfg=dict(num_gpus=1, num_procs=1)
     )
 ]

configs/models/vicuna/hf_vicuna_7b_v15_16k.py

@@ -17,6 +17,7 @@ models = [
         batch_size=8,
         model_kwargs=dict(device_map='auto'),
         batch_padding=False, # if false, inference with for-loop without batch padding
+        use_fastchat_template=True,
         run_cfg=dict(num_gpus=1, num_procs=1)
     )
 ]

configs/summarizers/groups/tydiqa.py

 tydiqa_summary_groups = []
 
 _tydiqa = ['arabic', 'bengali', 'english', 'finnish', 'indonesian', 'japanese', 'korean', 'russian', 'swahili', 'telugu', 'thai']
-_tydiqa = ['tyidqa-goldp_' + s for s in _tydiqa]
+_tydiqa = ['tydiqa-goldp_' + s for s in _tydiqa]
 tydiqa_summary_groups.append({'name': 'tydiqa-goldp', 'subsets': _tydiqa})

opencompass/datasets/cmb.py

@@ -18,6 +18,7 @@ class CMBDataset(BaseDataset):
         for d in val_data:
             d['option_str'] = '\n'.join(
                 [f'{k}. {v}' for k, v in d['option'].items() if len(v) > 1])
+            d['answer'] = 'NULL'
         val_dataset = Dataset.from_list(val_data)
 
         with open(osp.join(path, 'test.json'), 'r', encoding='utf-8') as f:
@@ -25,7 +26,6 @@ class CMBDataset(BaseDataset):
         for d in test_data:
             d['option_str'] = '\n'.join(
                 [f'{k}. {v}' for k, v in d['option'].items() if len(v) > 1])
-            d['answer'] = 'NULL'
         test_dataset = Dataset.from_list(test_data)
 
         return DatasetDict({'val': val_dataset, 'test': test_dataset})

opencompass/datasets/cmnli.py

@@ -7,6 +7,19 @@ from opencompass.registry import LOAD_DATASET
 from .base import BaseDataset
 
 
+@LOAD_DATASET.register_module()
+class cmnliDataset(BaseDataset):
+
+    @staticmethod
+    def load(path):
+        data = []
+        with open(path, 'r', encoding='utf-8') as f:
+            for line in f:
+                line = json.loads(line)
+                data.append(line)
+        return Dataset.from_list(data)
+
+
 @LOAD_DATASET.register_module()
 class cmnliDataset_V2(BaseDataset):
 

opencompass/datasets/longbench/longbench_lsht.py

 from datasets import Dataset, load_dataset
 
-from opencompass.registry import LOAD_DATASET
+from opencompass.registry import LOAD_DATASET, TEXT_POSTPROCESSORS
 
 from ..base import BaseDataset
 
@@ -28,3 +28,9 @@ class LongBenchlshtDataset(BaseDataset):
             })
         dataset[split] = Dataset.from_list(raw_data)
         return dataset
+
+
+@TEXT_POSTPROCESSORS.register_module()
+def lsht_postprocess(text: str) -> str:
+    text = text.lstrip('\n').split('\n')[0]
+    return text

opencompass/datasets/longbench/longbench_samsum.py

 from datasets import Dataset, load_dataset
 
-from opencompass.registry import LOAD_DATASET
+from opencompass.registry import LOAD_DATASET, TEXT_POSTPROCESSORS
 
 from ..base import BaseDataset
 
@@ -24,3 +24,9 @@ class LongBenchsamsumDataset(BaseDataset):
             })
         dataset[split] = Dataset.from_list(raw_data)
         return dataset
+
+
+@TEXT_POSTPROCESSORS.register_module()
+def samsum_postprocess(text: str) -> str:
+    text = text.lstrip('\n').split('\n')[0]
+    return text

opencompass/datasets/longbench/longbench_trec.py

 from datasets import Dataset, load_dataset
 
-from opencompass.registry import LOAD_DATASET
+from opencompass.registry import LOAD_DATASET, TEXT_POSTPROCESSORS
 
 from ..base import BaseDataset
 
@@ -28,3 +28,9 @@ class LongBenchtrecDataset(BaseDataset):
             })
         dataset[split] = Dataset.from_list(raw_data)
         return dataset
+
+
+@TEXT_POSTPROCESSORS.register_module()
+def trec_postprocess(text: str) -> str:
+    text = text.lstrip('\n').split('\n')[0]
+    return text