[Dataset] LongBench (#236)

Co-authored-by: wangchonghua <wangchonghua@pjlab.org.cn>

configs/datasets/longbench/longbenchnq/longbench_nq_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .longbench_nq_gen_d30cb9 import LongBench_nq_datasets  # noqa: F401, F403

configs/datasets/longbench/longbenchnq/longbench_nq_gen_d30cb9.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import LongBenchF1Evaluator, LongBenchnqDataset
+
+LongBench_nq_reader_cfg = dict(
+    input_columns=['context', 'input'],
+    output_column='answers',
+    train_split='test',
+    test_split='test'
+)
+
+LongBench_nq_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='Answer the question based on the given passage. Only give me the answer and do not output any other words. The following are some examples.\n\n{context}\n\n{input}'),
+            ], )),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=32)
+)
+
+LongBench_nq_eval_cfg = dict(
+    evaluator=dict(type=LongBenchF1Evaluator),
+    pred_role='BOT'
+)
+
+LongBench_nq_datasets = [
+    dict(
+        type=LongBenchnqDataset,
+        abbr='LongBench_nq',
+        path='THUDM/LongBench',
+        name='nq',
+        reader_cfg=LongBench_nq_reader_cfg,
+        infer_cfg=LongBench_nq_infer_cfg,
+        eval_cfg=LongBench_nq_eval_cfg)
+]

configs/datasets/longbench/longbenchpassage_count/longbench_passage_count_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .longbench_passage_count_gen_dcdaab import LongBench_passage_count_datasets  # noqa: F401, F403

configs/datasets/longbench/longbenchpassage_count/longbench_passage_count_gen_dcdaab.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import LongBenchCountEvaluator, LongBenchpassage_countDataset
+
+LongBench_passage_count_reader_cfg = dict(
+    input_columns=['context', 'input'],
+    output_column='answers',
+    train_split='test',
+    test_split='test'
+)
+
+LongBench_passage_count_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='There are some paragraphs below sourced from Wikipedia. Some of them may be duplicates. Please carefully read these paragraphs and determine how many unique paragraphs there are after removing duplicates. In other words, how many non-repeating paragraphs are there in total?\n\n{context}\n\nPlease enter the final count of unique paragraphs after removing duplicates. The output format should only contain the number, such as 1, 2, 3, and so on.\n\nThe final answer is: '),
+            ], )),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=32)
+)
+
+LongBench_passage_count_eval_cfg = dict(
+    evaluator=dict(type=LongBenchCountEvaluator),
+    pred_role='BOT'
+)
+
+LongBench_passage_count_datasets = [
+    dict(
+        type=LongBenchpassage_countDataset,
+        abbr='LongBench_passage_count',
+        path='THUDM/LongBench',
+        name='passage_count',
+        reader_cfg=LongBench_passage_count_reader_cfg,
+        infer_cfg=LongBench_passage_count_infer_cfg,
+        eval_cfg=LongBench_passage_count_eval_cfg)
+]

configs/datasets/longbench/longbenchpassage_retrieval_en/longbench_passage_retrieval_en_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .longbench_passage_retrieval_en_gen_734db5 import LongBench_passage_retrieval_en_datasets  # noqa: F401, F403

configs/datasets/longbench/longbenchpassage_retrieval_en/longbench_passage_retrieval_en_gen_734db5.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import LongBenchRetrievalEvaluator, LongBenchpassage_retrieval_enDataset
+
+LongBench_passage_retrieval_en_reader_cfg = dict(
+    input_columns=['context', 'input'],
+    output_column='answers',
+    train_split='test',
+    test_split='test'
+)
+
+LongBench_passage_retrieval_en_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='Here are 30 paragraphs from Wikipedia, along with an abstract. Please determine which paragraph the abstract is from.\n\n{context}\n\nThe following is an abstract.\n\n{input}\n\nPlease enter the number of the paragraph that the abstract is from. The answer format must be like \"Paragraph 1\", \"Paragraph 2\", etc.\n\nThe answer is: '),
+            ], )),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=32)
+)
+
+LongBench_passage_retrieval_en_eval_cfg = dict(
+    evaluator=dict(type=LongBenchRetrievalEvaluator),
+    pred_role='BOT'
+)
+
+LongBench_passage_retrieval_en_datasets = [
+    dict(
+        type=LongBenchpassage_retrieval_enDataset,
+        abbr='LongBench_passage_retrieval_en',
+        path='THUDM/LongBench',
+        name='passage_retrieval_en',
+        reader_cfg=LongBench_passage_retrieval_en_reader_cfg,
+        infer_cfg=LongBench_passage_retrieval_en_infer_cfg,
+        eval_cfg=LongBench_passage_retrieval_en_eval_cfg)
+]

configs/datasets/longbench/longbenchpassage_retrieval_zh/longbench_passage_retrieval_zh_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .longbench_passage_retrieval_zh_gen_01cca2 import LongBench_passage_retrieval_zh_datasets  # noqa: F401, F403

configs/datasets/longbench/longbenchpassage_retrieval_zh/longbench_passage_retrieval_zh_gen_01cca2.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import LongBenchRetrievalEvaluator, LongBenchpassage_retrieval_zhDataset
+
+LongBench_passage_retrieval_zh_reader_cfg = dict(
+    input_columns=['context', 'input'],
+    output_column='answers',
+    train_split='test',
+    test_split='test'
+)
+
+LongBench_passage_retrieval_zh_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='以下是若干段落文字，以及其中一个段落的摘要。请确定给定的摘要出自哪一段。\n\n{context}\n\n下面是一个摘要\n\n{input}\n\n请输入摘要所属段落的编号。答案格式必须是\"段落1\"，\"段落2\"等格式\n\n答案是：'),
+            ], )),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=32)
+)
+
+LongBench_passage_retrieval_zh_eval_cfg = dict(
+    evaluator=dict(type=LongBenchRetrievalEvaluator, language='zh'),
+    pred_role='BOT'
+)
+
+LongBench_passage_retrieval_zh_datasets = [
+    dict(
+        type=LongBenchpassage_retrieval_zhDataset,
+        abbr='LongBench_passage_retrieval_zh',
+        path='THUDM/LongBench',
+        name='passage_retrieval_zh',
+        reader_cfg=LongBench_passage_retrieval_zh_reader_cfg,
+        infer_cfg=LongBench_passage_retrieval_zh_infer_cfg,
+        eval_cfg=LongBench_passage_retrieval_zh_eval_cfg)
+]

configs/datasets/longbench/longbenchqasper/longbench_qasper_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .longbench_qasper_gen_6b3efc import LongBench_qasper_datasets  # noqa: F401, F403

configs/datasets/longbench/longbenchqasper/longbench_qasper_gen_6b3efc.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import LongBenchF1Evaluator, LongBenchqasperDataset
+
+LongBench_qasper_reader_cfg = dict(
+    input_columns=['context', 'input'],
+    output_column='answers',
+    train_split='test',
+    test_split='test'
+)
+
+LongBench_qasper_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:'),
+            ], )),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=32)
+)
+
+LongBench_qasper_eval_cfg = dict(
+    evaluator=dict(type=LongBenchF1Evaluator),
+    pred_role='BOT'
+)
+
+LongBench_qasper_datasets = [
+    dict(
+        type=LongBenchqasperDataset,
+        abbr='LongBench_qasper',
+        path='THUDM/LongBench',
+        name='qasper',
+        reader_cfg=LongBench_qasper_reader_cfg,
+        infer_cfg=LongBench_qasper_infer_cfg,
+        eval_cfg=LongBench_qasper_eval_cfg)
+]

configs/datasets/longbench/longbenchqmsum/longbench_qmsum_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .longbench_qmsum_gen_d33331 import LongBench_qmsum_datasets  # noqa: F401, F403

configs/datasets/longbench/longbenchqmsum/longbench_qmsum_gen_d33331.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import LongBenchRougeEvaluator, LongBenchqmsumDataset
+
+LongBench_qmsum_reader_cfg = dict(
+    input_columns=['context', 'input'],
+    output_column='answers',
+    train_split='test',
+    test_split='test'
+)
+
+LongBench_qmsum_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='You are given a meeting transcript and a query containing a question or instruction. Answer the query in one or more sentences.\n\nTranscript:\n{context}\n\nNow, answer the query based on the above meeting transcript in one or more sentences.\n\nQuery: {input}\nAnswer:'),
+            ], )),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=512)
+)
+
+LongBench_qmsum_eval_cfg = dict(
+    evaluator=dict(type=LongBenchRougeEvaluator),
+    pred_role='BOT'
+)
+
+LongBench_qmsum_datasets = [
+    dict(
+        type=LongBenchqmsumDataset,
+        abbr='LongBench_qmsum',
+        path='THUDM/LongBench',
+        name='qmsum',
+        reader_cfg=LongBench_qmsum_reader_cfg,
+        infer_cfg=LongBench_qmsum_infer_cfg,
+        eval_cfg=LongBench_qmsum_eval_cfg)
+]

configs/datasets/longbench/longbenchrepobench/longbench_repobench_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .longbench_repobench_gen_6df953 import LongBench_repobench_datasets  # noqa: F401, F403

configs/datasets/longbench/longbenchrepobench/longbench_repobench_gen_6df953.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import LongBenchCodeSimEvaluator, LongBenchrepobenchDataset
+
+LongBench_repobench_reader_cfg = dict(
+    input_columns=['context', 'input'],
+    output_column='answers',
+    train_split='test',
+    test_split='test'
+)
+
+LongBench_repobench_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='Please complete the code given below. \n{context}{input}Next line of code:\n'),
+            ], )),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=64)
+)
+
+LongBench_repobench_eval_cfg = dict(
+    evaluator=dict(type=LongBenchCodeSimEvaluator),
+    pred_role='BOT'
+)
+
+LongBench_repobench_datasets = [
+    dict(
+        type=LongBenchrepobenchDataset,
+        abbr='LongBench_repobench-p',
+        path='THUDM/LongBench',
+        name='repobench-p',
+        reader_cfg=LongBench_repobench_reader_cfg,
+        infer_cfg=LongBench_repobench_infer_cfg,
+        eval_cfg=LongBench_repobench_eval_cfg)
+]

configs/datasets/longbench/longbenchtrec/longbench_trec_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .longbench_trec_gen_824187 import LongBench_trec_datasets  # noqa: F401, F403

configs/datasets/longbench/longbenchtrec/longbench_trec_gen_824187.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import LongBenchClassificationEvaluator, LongBenchtrecDataset
+
+LongBench_trec_reader_cfg = dict(
+    input_columns=['context', 'input'],
+    output_column='all_labels',
+    train_split='test',
+    test_split='test'
+)
+
+LongBench_trec_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='Please determine the type of the question below. Here are some examples of questions.\n\n{context}\n{input}'),
+            ], )),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=64)
+)
+
+LongBench_trec_eval_cfg = dict(
+    evaluator=dict(type=LongBenchClassificationEvaluator),
+    pred_role='BOT'
+)
+
+LongBench_trec_datasets = [
+    dict(
+        type=LongBenchtrecDataset,
+        abbr='LongBench_trec',
+        path='THUDM/LongBench',
+        name='trec',
+        reader_cfg=LongBench_trec_reader_cfg,
+        infer_cfg=LongBench_trec_infer_cfg,
+        eval_cfg=LongBench_trec_eval_cfg)
+]

configs/datasets/longbench/longbenchtriviaqa/longbench_triviaqa_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .longbench_triviaqa_gen_d30cb9 import LongBench_triviaqa_datasets  # noqa: F401, F403

configs/datasets/longbench/longbenchtriviaqa/longbench_triviaqa_gen_d30cb9.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import LongBenchF1Evaluator, LongBenchtriviaqaDataset
+
+LongBench_triviaqa_reader_cfg = dict(
+    input_columns=['context', 'input'],
+    output_column='answers',
+    train_split='test',
+    test_split='test'
+)
+
+LongBench_triviaqa_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='Answer the question based on the given passage. Only give me the answer and do not output any other words. The following are some examples.\n\n{context}\n\n{input}'),
+            ], )),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=32)
+)
+
+LongBench_triviaqa_eval_cfg = dict(
+    evaluator=dict(type=LongBenchF1Evaluator),
+    pred_role='BOT'
+)
+
+LongBench_triviaqa_datasets = [
+    dict(
+        type=LongBenchtriviaqaDataset,
+        abbr='LongBench_triviaqa',
+        path='THUDM/LongBench',
+        name='triviaqa',
+        reader_cfg=LongBench_triviaqa_reader_cfg,
+        infer_cfg=LongBench_triviaqa_infer_cfg,
+        eval_cfg=LongBench_triviaqa_eval_cfg)
+]

configs/datasets/longbench/longbenchvcsum/longbench_vcsum_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .longbench_vcsum_gen_f7a8ac import LongBench_vcsum_datasets  # noqa: F401, F403

configs/datasets/longbench/longbenchvcsum/longbench_vcsum_gen_f7a8ac.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import LongBenchRougeEvaluator, LongBenchvcsumDataset
+
+LongBench_vcsum_reader_cfg = dict(
+    input_columns=['context'],
+    output_column='answers',
+    train_split='test',
+    test_split='test'
+)
+
+LongBench_vcsum_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='下面有一段会议记录，请你阅读后，写一段总结，总结会议的内容。\n会议记录：\n{context}\n\n会议总结：'),
+            ], )),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=512)
+)
+
+LongBench_vcsum_eval_cfg = dict(
+    evaluator=dict(type=LongBenchRougeEvaluator, language='zh'),
+    pred_role='BOT'
+)
+
+LongBench_vcsum_datasets = [
+    dict(
+        type=LongBenchvcsumDataset,
+        abbr='LongBench_vcsum',
+        path='THUDM/LongBench',
+        name='vcsum',
+        reader_cfg=LongBench_vcsum_reader_cfg,
+        infer_cfg=LongBench_vcsum_infer_cfg,
+        eval_cfg=LongBench_vcsum_eval_cfg)
+]