[Feature] Add LEval datasets

Co-authored-by: kennymckormick <dhd@pku.edu.cn>

opencompass/datasets/LEval_gsm100.py

+from datasets import Dataset, load_dataset
+
+from opencompass.registry import LOAD_DATASET, TEXT_POSTPROCESSORS
+
+from .base import BaseDataset
+
+
+@TEXT_POSTPROCESSORS.register_module('gsm100_dataset')
+def gsm100_dataset_postprocess(text: str) -> str:
+    return text.replace(',', '')
+
+
+@TEXT_POSTPROCESSORS.register_module('gsm100')
+def gsm100_postprocess(text: str) -> str:
+    # text = text.split('\n\n')[0]
+    segs = text.split('The answer is')
+    if len(segs) < 2:
+        return ''
+    text = segs[1]
+    text = text.split(' ')
+    flag = False
+    ret = ''
+    for i in range(len(text)):
+        s = text[i]
+        for i in range(len(s)):
+            if s[i].isdigit():
+                flag = True
+                ret = s
+                break
+        if flag:
+            break
+    ret1 = ''
+    for i in range(len(ret)):
+        if ret[i].isdigit():
+            ret1 += ret[i]
+    return ret1
+
+
+@LOAD_DATASET.register_module()
+class LEvalGSM100Dataset(BaseDataset):
+
+    @staticmethod
+    def load(**kwargs):
+        dataset = load_dataset(**kwargs)
+        split = 'test'
+        raw_data = []
+        for i in range(len(dataset[split])):
+            instructions = dataset[split]['instructions'][i]
+            outputs = dataset[split]['outputs'][i]
+            context = dataset[split]['input'][i]
+            for question, answer in zip(instructions, outputs):
+                raw_data.append({
+                    'question': question,
+                    'context': context,
+                    'answer': answer
+                })
+        dataset[split] = Dataset.from_list(raw_data)
+        return dataset

opencompass/datasets/LEval_legal_contract_qa.py

+from datasets import Dataset, load_dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class LEvalLegalContractQADataset(BaseDataset):
+
+    @staticmethod
+    def load(**kwargs):
+        dataset = load_dataset(**kwargs)
+        split = 'test'
+        raw_data = []
+        for i in range(len(dataset[split])):
+            instructions = dataset[split]['instructions'][i]
+            outputs = dataset[split]['outputs'][i]
+            context = dataset[split]['input'][i]
+            for question, answer in zip(instructions, outputs):
+                raw_data.append({
+                    'question': question,
+                    'context': context,
+                    'answer': answer
+                })
+        dataset[split] = Dataset.from_list(raw_data)
+        return dataset

opencompass/datasets/LEval_meeting_summ.py

+from datasets import Dataset, load_dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class LEvalMeetingSummDataset(BaseDataset):
+
+    @staticmethod
+    def load(**kwargs):
+        dataset = load_dataset(**kwargs)
+        split = 'test'
+        raw_data = []
+        for i in range(len(dataset[split])):
+            instructions = dataset[split]['instructions'][i]
+            outputs = dataset[split]['outputs'][i]
+            context = dataset[split]['input'][i]
+            for question, answer in zip(instructions, outputs):
+                raw_data.append({
+                    'question': question,
+                    'context': context,
+                    'answer': answer
+                })
+        dataset[split] = Dataset.from_list(raw_data)
+        return dataset

opencompass/datasets/LEval_multidoc_qa.py

+from datasets import Dataset, load_dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class LEvalMultidocQADataset(BaseDataset):
+
+    @staticmethod
+    def load(**kwargs):
+        dataset = load_dataset(**kwargs)
+        split = 'test'
+        raw_data = []
+        for i in range(len(dataset[split])):
+            instructions = dataset[split]['instructions'][i]
+            outputs = dataset[split]['outputs'][i]
+            context = dataset[split]['input'][i]
+            for question, answer in zip(instructions, outputs):
+                raw_data.append({
+                    'question': question,
+                    'context': context,
+                    'answer': answer
+                })
+        dataset[split] = Dataset.from_list(raw_data)
+        return dataset

opencompass/datasets/LEval_narrattive_qa.py

+from datasets import Dataset, load_dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class LEvalNarrativeQADataset(BaseDataset):
+
+    @staticmethod
+    def load(**kwargs):
+        dataset = load_dataset(**kwargs)
+        split = 'test'
+        raw_data = []
+        for i in range(len(dataset[split])):
+            instructions = dataset[split]['instructions'][i]
+            outputs = dataset[split]['outputs'][i]
+            context = dataset[split]['input'][i]
+            for question, answer in zip(instructions, outputs):
+                raw_data.append({
+                    'question': question,
+                    'context': context,
+                    'answer': answer
+                })
+        dataset[split] = Dataset.from_list(raw_data)
+        return dataset

opencompass/datasets/LEval_natural_question.py

+from datasets import Dataset, load_dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class LEvalNaturalQuestionDataset(BaseDataset):
+
+    @staticmethod
+    def load(**kwargs):
+        dataset = load_dataset(**kwargs)
+        split = 'test'
+        raw_data = []
+        for i in range(len(dataset[split])):
+            instructions = dataset[split]['instructions'][i]
+            outputs = dataset[split]['outputs'][i]
+            context = dataset[split]['input'][i]
+            for question, answer in zip(instructions, outputs):
+                raw_data.append({
+                    'question': question,
+                    'context': context,
+                    'answer': answer
+                })
+        dataset[split] = Dataset.from_list(raw_data)
+        return dataset

opencompass/datasets/LEval_news_summ.py

+from datasets import Dataset, load_dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class LEvalNewsSummDataset(BaseDataset):
+
+    @staticmethod
+    def load(**kwargs):
+        dataset = load_dataset(**kwargs)
+        split = 'test'
+        raw_data = []
+        for i in range(len(dataset[split])):
+            instructions = dataset[split]['instructions'][i]
+            outputs = dataset[split]['outputs'][i]
+            context = dataset[split]['input'][i]
+            for question, answer in zip(instructions, outputs):
+                raw_data.append({
+                    'question': question,
+                    'context': context,
+                    'answer': answer
+                })
+        dataset[split] = Dataset.from_list(raw_data)
+        return dataset

opencompass/datasets/LEval_paper_assistant.py

+from datasets import Dataset, load_dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class LEvalPaperAssistantDataset(BaseDataset):
+
+    @staticmethod
+    def load(**kwargs):
+        dataset = load_dataset(**kwargs)
+        split = 'test'
+        raw_data = []
+        for i in range(len(dataset[split])):
+            instructions = dataset[split]['instructions'][i]
+            outputs = dataset[split]['outputs'][i]
+            context = dataset[split]['input'][i]
+            for question, answer in zip(instructions, outputs):
+                raw_data.append({
+                    'question': question,
+                    'context': context,
+                    'answer': answer
+                })
+        dataset[split] = Dataset.from_list(raw_data)
+        return dataset

opencompass/datasets/LEval_patent_summ.py

+from datasets import Dataset, load_dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class LEvalPatentSummDataset(BaseDataset):
+
+    @staticmethod
+    def load(**kwargs):
+        dataset = load_dataset(**kwargs)
+        split = 'test'
+        raw_data = []
+        for i in range(len(dataset[split])):
+            instructions = dataset[split]['instructions'][i]
+            outputs = dataset[split]['outputs'][i]
+            context = dataset[split]['input'][i]
+            for question, answer in zip(instructions, outputs):
+                raw_data.append({
+                    'question': question,
+                    'context': context,
+                    'answer': answer
+                })
+        dataset[split] = Dataset.from_list(raw_data)
+        return dataset

opencompass/datasets/LEval_quality.py

+from datasets import Dataset, load_dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class LEvalQualityDataset(BaseDataset):
+
+    @staticmethod
+    def load(**kwargs):
+        dataset = load_dataset(**kwargs)
+        split = 'test'
+        raw_data = []
+        for i in range(len(dataset[split])):
+            instructions = dataset[split]['instructions'][i]
+            outputs = dataset[split]['outputs'][i]
+            context = dataset[split]['input'][i]
+            for question, answer in zip(instructions, outputs):
+                raw_data.append({
+                    'question': question,
+                    'context': context,
+                    'answer': answer[1]
+                })
+        dataset[split] = Dataset.from_list(raw_data)
+        return dataset

opencompass/datasets/LEval_review_summ.py

+from datasets import Dataset, load_dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class LEvalReviewSummDataset(BaseDataset):
+
+    @staticmethod
+    def load(**kwargs):
+        dataset = load_dataset(**kwargs)
+        split = 'test'
+        raw_data = []
+        for i in range(len(dataset[split])):
+            instructions = dataset[split]['instructions'][i]
+            outputs = dataset[split]['outputs'][i]
+            context = dataset[split]['input'][i]
+            for question, answer in zip(instructions, outputs):
+                raw_data.append({
+                    'question': question,
+                    'context': context,
+                    'answer': answer
+                })
+        dataset[split] = Dataset.from_list(raw_data)
+        return dataset

opencompass/datasets/LEval_scientific_qa.py

+from datasets import Dataset, load_dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class LEvalScientificQADataset(BaseDataset):
+
+    @staticmethod
+    def load(**kwargs):
+        dataset = load_dataset(**kwargs)
+        split = 'test'
+        raw_data = []
+        for i in range(len(dataset[split])):
+            instructions = dataset[split]['instructions'][i]
+            outputs = dataset[split]['outputs'][i]
+            context = dataset[split]['input'][i]
+            for question, answer in zip(instructions, outputs):
+                raw_data.append({
+                    'question': question,
+                    'context': context,
+                    'answer': answer
+                })
+        dataset[split] = Dataset.from_list(raw_data)
+        return dataset

opencompass/datasets/LEval_topic_retrieval.py

+from datasets import Dataset, load_dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class LEvalTopicRetrievalDataset(BaseDataset):
+
+    @staticmethod
+    def load(**kwargs):
+        dataset = load_dataset(**kwargs)
+        split = 'test'
+        raw_data = []
+        for i in range(len(dataset[split])):
+            instructions = dataset[split]['instructions'][i]
+            outputs = dataset[split]['outputs'][i]
+            context = dataset[split]['input'][i]
+            for question, answer in zip(instructions, outputs):
+                raw_data.append({
+                    'question': question,
+                    'context': context,
+                    'answer': answer
+                })
+        dataset[split] = Dataset.from_list(raw_data)
+        return dataset

opencompass/datasets/LEval_tpo.py

+from datasets import Dataset, load_dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class LEvalTPODataset(BaseDataset):
+
+    @staticmethod
+    def load(**kwargs):
+        dataset = load_dataset(**kwargs)
+        split = 'test'
+        raw_data = []
+        for i in range(len(dataset[split])):
+            instructions = dataset[split]['instructions'][i]
+            outputs = dataset[split]['outputs'][i]
+            context = dataset[split]['input'][i]
+            for question, answer in zip(instructions, outputs):
+                raw_data.append({
+                    'question': question,
+                    'context': context,
+                    'answer': answer
+                })
+        dataset[split] = Dataset.from_list(raw_data)
+        return dataset

opencompass/datasets/LEval_tvshow_summ.py

+from datasets import Dataset, load_dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class LEvalTVShowSummDataset(BaseDataset):
+
+    @staticmethod
+    def load(**kwargs):
+        dataset = load_dataset(**kwargs)
+        split = 'test'
+        raw_data = []
+        for i in range(len(dataset[split])):
+            instructions = dataset[split]['instructions'][i]
+            outputs = dataset[split]['outputs'][i]
+            context = dataset[split]['input'][i]
+            for question, answer in zip(instructions, outputs):
+                raw_data.append({
+                    'question': question,
+                    'context': context,
+                    'answer': answer
+                })
+        dataset[split] = Dataset.from_list(raw_data)
+        return dataset

opencompass/datasets/__init__.py

@@ -34,6 +34,24 @@ from .iwslt2017 import *  # noqa: F401, F403
 from .jigsawmultilingual import *  # noqa: F401, F403
 from .lambada import *  # noqa: F401, F403
 from .lcsts import *  # noqa: F401, F403
+from .LEval_coursera import *  # noqa: F401, F403
+from .LEval_financial_qa import *  # noqa: F401, F403
+from .LEval_gov_report_summ import *  # noqa: F401, F403
+from .LEval_gsm100 import *  # noqa: F401, F403
+from .LEval_legal_contract_qa import *  # noqa: F401, F403
+from .LEval_meeting_summ import *  # noqa: F401, F403
+from .LEval_multidoc_qa import *  # noqa: F401, F403
+from .LEval_narrattive_qa import *  # noqa: F401, F403
+from .LEval_natural_question import *  # noqa: F401, F403
+from .LEval_news_summ import *  # noqa: F401, F403
+from .LEval_paper_assistant import *  # noqa: F401, F403
+from .LEval_patent_summ import *  # noqa: F401, F403
+from .LEval_quality import *  # noqa: F401, F403
+from .LEval_review_summ import *  # noqa: F401, F403
+from .LEval_scientific_qa import *  # noqa: F401, F403
+from .LEval_topic_retrieval import *  # noqa: F401, F403
+from .LEval_tpo import *  # noqa: F401, F403
+from .LEval_tvshow_summ import *  # noqa: F401, F403
 from .math import *  # noqa: F401, F403
 from .mbpp import *  # noqa: F401, F403
 from .mmlu import *  # noqa: F401, F403