Support a batch of datasets.

configs/datasets/siqa/siqa_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .siqa_gen_a3c714 import siqa_datasets  # noqa: F401, F403

configs/datasets/storycloze/storycloze_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .storycloze_gen_c5a230 import storycloze_datasets  # noqa: F401, F403

configs/datasets/summedits/summedits_ppl.py

+from mmengine.config import read_base
+
+with read_base():
+    from .summedits_ppl_163352 import summedits_datasets  # noqa: F401, F403

configs/datasets/triviaqa/triviaqa_gen_4f9838.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import TriviaQADataset, TriviaQAEvaluator
+
+triviaqa_reader_cfg = dict(
+    input_columns=['question'],
+    output_column='answer',
+    train_split='dev',
+    test_split='dev')
+
+triviaqa_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='Answer these questions:\nQ: {question}?'),
+                dict(role='BOT', prompt='A:'),
+            ], )),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=50))
+
+triviaqa_eval_cfg = dict(
+    evaluator=dict(type=TriviaQAEvaluator), pred_role='BOT')
+
+triviaqa_datasets = [
+    dict(
+        type=TriviaQADataset,
+        abbr='triviaqa',
+        path='./data/triviaqa/',
+        reader_cfg=triviaqa_reader_cfg,
+        infer_cfg=triviaqa_infer_cfg,
+        eval_cfg=triviaqa_eval_cfg)
+]

configs/datasets/triviaqa/triviaqa_gen_5ef393.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import TriviaQADataset, TriviaQAEvaluator
+
+triviaqa_reader_cfg = dict(
+    input_columns=['question'],
+    output_column='answer',
+    train_split='dev',
+    test_split='dev')
+
+triviaqa_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template='Answer these questions:\nQ: {question}\nA:{answer}'),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=50))
+
+triviaqa_eval_cfg = dict(
+    evaluator=dict(type=TriviaQAEvaluator), pred_role='BOT')
+
+triviaqa_datasets = [
+    dict(
+        type=TriviaQADataset,
+        abbr='triviaqa',
+        path='./data/triviaqa/',
+        reader_cfg=triviaqa_reader_cfg,
+        infer_cfg=triviaqa_infer_cfg,
+        eval_cfg=triviaqa_eval_cfg)
+]

configs/datasets/winograd/winograd_ppl_c21c87.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import winogradDataset
+
+winograd_reader_cfg = dict(
+    input_columns=['opt1', 'opt2'],
+    output_column='label',
+    train_split='test',
+    test_split='test')
+
+winograd_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            0:
+            "{prompt} Q: In the previous text, what does '{pronoun}' refer to? A: {opt1}.",  # noqa
+            1:
+            "{prompt} Q: In the previous text, what does '{pronoun}' refer to? A: {opt2}.",  # noqa
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+winograd_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )
+
+winograd_datasets = [
+    dict(
+        abbr='winograd',
+        type=winogradDataset,
+        path='winograd_wsc',
+        name='wsc285',
+        reader_cfg=winograd_reader_cfg,
+        infer_cfg=winograd_infer_cfg,
+        eval_cfg=winograd_eval_cfg)
+]

configs/datasets/winogrande/winogrande_ppl_515f92.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import winograndeDataset
+
+winogrande_reader_cfg = dict(
+    input_columns=['opt1', 'opt2'],
+    output_column='answer',
+    train_split='validation',
+    test_split='validation')
+
+winogrande_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            1: "Good sentence: {opt1}",
+            2: "Good sentence: {opt2}",
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+winogrande_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )
+
+winogrande_datasets = [
+    dict(
+        abbr='winogrande',
+        type=winograndeDataset,
+        path='winogrande',
+        name='winogrande_xs',
+        reader_cfg=winogrande_reader_cfg,
+        infer_cfg=winogrande_infer_cfg,
+        eval_cfg=winogrande_eval_cfg)
+]

configs/summarizers/groups/ceval.py

+ceval_summary_groups = []
+
+_ceval_stem = ['computer_network', 'operating_system', 'computer_architecture', 'college_programming', 'college_physics', 'college_chemistry', 'advanced_mathematics', 'probability_and_statistics', 'discrete_mathematics', 'electrical_engineer', 'metrology_engineer', 'high_school_mathematics', 'high_school_physics', 'high_school_chemistry', 'high_school_biology', 'middle_school_mathematics', 'middle_school_biology', 'middle_school_physics', 'middle_school_chemistry', 'veterinary_medicine']
+_ceval_stem = ['ceval-' + s for s in _ceval_stem]
+ceval_summary_groups.append({'name': 'ceval-stem', 'subsets': _ceval_stem})
+
+_ceval_social_science = ['college_economics', 'business_administration', 'marxism', 'mao_zedong_thought', 'education_science', 'teacher_qualification', 'high_school_politics', 'high_school_geography', 'middle_school_politics', 'middle_school_geography']
+_ceval_social_science = ['ceval-' + s for s in _ceval_social_science]
+ceval_summary_groups.append({'name': 'ceval-social-science', 'subsets': _ceval_social_science})
+
+_ceval_humanities = ['modern_chinese_history', 'ideological_and_moral_cultivation', 'logic', 'law', 'chinese_language_and_literature', 'art_studies', 'professional_tour_guide', 'legal_professional', 'high_school_chinese', 'high_school_history', 'middle_school_history']
+_ceval_humanities = ['ceval-' + s for s in _ceval_humanities]
+ceval_summary_groups.append({'name': 'ceval-humanities', 'subsets': _ceval_humanities})
+
+_ceval_other = ['civil_servant', 'sports_science', 'plant_protection', 'basic_medicine', 'clinical_medicine', 'urban_and_rural_planner', 'accountant', 'fire_engineer', 'environmental_impact_assessment_engineer', 'tax_accountant', 'physician']
+_ceval_other = ['ceval-' + s for s in _ceval_other]
+ceval_summary_groups.append({'name': 'ceval-other', 'subsets': _ceval_other})
+
+_ceval_hard = ['advanced_mathematics', 'discrete_mathematics', 'probability_and_statistics', 'college_chemistry', 'college_physics', 'high_school_mathematics', 'high_school_chemistry', 'high_school_physics']
+_ceval_hard = ['ceval-' + s for s in _ceval_hard]
+ceval_summary_groups.append({'name': 'ceval-hard', 'subsets': _ceval_hard})
+
+_ceval_all = _ceval_stem + _ceval_social_science + _ceval_humanities + _ceval_other
+ceval_summary_groups.append({'name': 'ceval', 'subsets': _ceval_all})

opencompass/datasets/ceval.py

+import os.path as osp
+
+from datasets import DatasetDict, load_dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class CEvalDataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str, name: str):
+        dev_dataset = load_dataset(
+            'csv',
+            data_files=osp.join(path, 'dev', f'{name}_dev.csv'),
+            split='train')
+        val_dataset = load_dataset(
+            'csv',
+            data_files=osp.join(path, 'val', f'{name}_val.csv'),
+            split='train')
+        val_dataset = val_dataset.add_column('explanation',
+                                             [''] * len(val_dataset))
+        test_dataset = load_dataset(
+            'csv',
+            data_files=osp.join(path, 'test', f'{name}_test.csv'),
+            split='train')
+        test_dataset = test_dataset.add_column(
+            'answer',
+            [''] * len(test_dataset)).add_column('explanation',
+                                                 [''] * len(test_dataset))
+        return DatasetDict({
+            'val': val_dataset,
+            'dev': dev_dataset,
+            'test': test_dataset
+        })

opencompass/datasets/csl.py

+import json
+
+from datasets import Dataset, load_dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class CslDataset(BaseDataset):
+
+    @staticmethod
+    def load(**kwargs):
+
+        dataset = load_dataset(**kwargs)
+
+        def preprocess(example):
+            keywords = '，'.join(example['keyword'])
+            example['keywords'] = keywords
+
+            return example
+
+        dataset = dataset.map(preprocess)
+        return dataset
+
+
+@LOAD_DATASET.register_module()
+class CslDataset_V2(BaseDataset):
+
+    @staticmethod
+    def load(path):
+        data = []
+        with open(path, 'r') as f:
+            for line in f:
+                line = json.loads(line)
+                item = {
+                    'abst': line['abst'],
+                    'keywords': '，'.join(line['keyword']),
+                    'label': 'AB'[int(line['label'])],
+                }
+                data.append(item)
+        return Dataset.from_list(data)

opencompass/datasets/drop.py

+from datasets import DatasetDict, load_dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class dropDataset(BaseDataset):
+
+    @staticmethod
+    def load(**kwargs):
+        dataset = load_dataset(**kwargs, split='validation')
+
+        def pre_process(example):
+            example['answers'] = example['answers_spans']['spans']
+            example['prompt'] = example.pop('passage')
+            return example
+
+        def only_number(example):
+            for i in example['answers_spans']['types']:
+                if i == 'number':
+                    return True
+            return False
+
+        dataset = dataset.filter(only_number)
+        dataset = dataset.map(pre_process).remove_columns(
+            ['section_id', 'query_id'])
+        return DatasetDict({'validation': dataset})

opencompass/datasets/gsm8k.py

+from opencompass.registry import TEXT_POSTPROCESSORS
+
+
+@TEXT_POSTPROCESSORS.register_module('gsm8k_dataset')
+def gsm8k_dataset_postprocess(text: str) -> str:
+    return text.split('#### ')[1].replace(',', '')
+
+
+@TEXT_POSTPROCESSORS.register_module('gsm8k')
+def gsm8k_postprocess(text: str) -> str:
+    text = text.split('\n\n')[0]
+    text = text.split(' ')[::-1]
+    flag = False
+    ret = ''
+    for i in range(len(text)):
+        s = text[i]
+        for i in range(len(s)):
+            if s[i].isdigit():
+                flag = True
+                ret = s
+                break
+        if flag:
+            break
+    ret1 = ''
+    for i in range(len(ret)):
+        if ret[i].isdigit():
+            ret1 += ret[i]
+    return ret1

opencompass/datasets/lcsts.py

+import os.path as osp
+
+from datasets import Dataset
+
+from opencompass.registry import LOAD_DATASET, TEXT_POSTPROCESSORS
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class LCSTSDataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str):
+        src_path = osp.join(path, 'test.src.txt')
+        tgt_path = osp.join(path, 'test.tgt.txt')
+
+        src_lines = open(src_path, 'r', encoding='utf-8').readlines()
+        tgt_lines = open(tgt_path, 'r', encoding='utf-8').readlines()
+
+        data = {'content': [], 'abst': []}
+
+        for _, (src_text, tgt_text) in enumerate(zip(src_lines, tgt_lines)):
+            data['content'].append(src_text.strip())
+            data['abst'].append(tgt_text.strip())
+
+        dataset = Dataset.from_dict({
+            'content': data['content'],
+            'abst': data['abst']
+        })
+        return dataset
+
+
+@TEXT_POSTPROCESSORS.register_module('lcsts')
+def lcsts_postprocess(text: str) -> str:
+    text = text.strip().split('\n')[0].replace('своей', '').strip()
+    text = text.replace('1. ', '') if text.startswith('1. ') else text
+    text = text.replace('- ', '') if text.startswith('- ') else text
+    text = text.strip('“，。！”')
+    return text

opencompass/datasets/obqa.py

+from datasets import load_dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class OBQADataset(BaseDataset):
+
+    @staticmethod
+    def load(**kwargs):
+        dataset = load_dataset(**kwargs)
+
+        def pre_process(example):
+            for i in range(4):
+                example[chr(ord('A') + i)] = example['choices']['text'][i]
+            return example
+
+        dataset = dataset.map(pre_process).remove_columns(['id', 'choices'])
+        return dataset

opencompass/datasets/summscreen.py

+from datasets import Dataset, DatasetDict
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class SummScreenDataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str):
+        import json
+        import os
+        dataset_dict = DatasetDict()
+        split = 'dev'
+        dev_list = []
+        fd_folder = os.path.join(path, 'SummScreen_raw', 'fd')
+        files = os.listdir(fd_folder)
+        for file in files:
+            filename = os.path.join(fd_folder, file)
+            with open(filename, 'r') as f:
+                data = json.load(f)
+                summary = ''.join(data['Recap'])
+                content = '\n'.join(data['Transcript'])
+                dev_list.append({
+                    'content': content,
+                    'summary': summary,
+                })
+
+        tms_folder = os.path.join(path, 'SummScreen_raw', 'tms')
+        files = os.listdir(tms_folder)
+        for file in files:
+            filename = os.path.join(tms_folder, file)
+            with open(filename, 'r') as f:
+                data = json.load(f)
+                summary = ''.join(data['Recap'])
+                content = '\n'.join(data['Transcript'])
+                dev_list.append({
+                    'content': content,
+                    'summary': summary,
+                })
+        dataset_dict[split] = Dataset.from_list(dev_list)
+        return dataset_dict

opencompass/datasets/wic.py

+import json
+
+from datasets import Dataset, load_dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class WiCDataset(BaseDataset):
+
+    @staticmethod
+    def load(**kwargs):
+
+        dataset = load_dataset(**kwargs)
+
+        def preprocess(example):
+            if example['label'] == 'true':
+                example['answer'] = 1
+            else:
+                example['answer'] = 0
+
+            return example
+
+        dataset = dataset.map(preprocess)
+        return dataset
+
+
+@LOAD_DATASET.register_module()
+class WiCDataset_V2(BaseDataset):
+
+    @staticmethod
+    def load(path):
+        dataset = []
+        with open(path, 'r') as f:
+            for line in f:
+                line = json.loads(line)
+                line['label'] = {'true': 'A', 'false': 'B'}[line['label']]
+                dataset.append(line)
+        return Dataset.from_list(dataset)

opencompass/openicl/__init__.py

+from .icl_dataset_reader import DatasetReader
+from .icl_evaluator import *
+from .icl_prompt_template import PromptTemplate
+from .icl_retriever import *
+from .icl_inferencer import *