[Feat] support opencompass

configs/datasets/narrativeqa/narrativeqa_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .narrativeqa_gen_5786a7 import narrativeqa_datasets  # noqa: F401, F403

configs/datasets/nq/nq_gen_c00b89.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import NaturalQuestionDataset, NQEvaluator
+
+nq_reader_cfg = dict(
+    input_columns=['question'], output_column='answer', train_split='test')
+
+nq_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='Question: {question}?\nAnswer: '),
+            ], )),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+nq_eval_cfg = dict(evaluator=dict(type=NQEvaluator), pred_role="BOT")
+
+nq_datasets = [
+    dict(
+        type=NaturalQuestionDataset,
+        abbr='nq',
+        path='./data/nq/',
+        reader_cfg=nq_reader_cfg,
+        infer_cfg=nq_infer_cfg,
+        eval_cfg=nq_eval_cfg)
+]

configs/datasets/piqa/piqa_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .piqa_gen_8287ae import piqa_datasets  # noqa: F401, F403

configs/datasets/piqa/piqa_ppl_788dbe.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import HFDataset
+
+piqa_reader_cfg = dict(
+    input_columns=['goal', 'sol1', 'sol2'],
+    output_column='label',
+    test_split='validation')
+
+piqa_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            0: 'The following makes sense: \nQ: {goal}\nA: {sol1}\n',
+            1: 'The following makes sense: \nQ: {goal}\nA: {sol2}\n'
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+piqa_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+piqa_datasets = [
+    dict(
+        type=HFDataset,
+        path='piqa',
+        reader_cfg=piqa_reader_cfg,
+        infer_cfg=piqa_infer_cfg,
+        eval_cfg=piqa_eval_cfg)
+]

configs/datasets/qabench/qabench_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .qabench_gen_0d5967 import qabench_datasets  # noqa: F401, F403

configs/datasets/race/race_gen_12de48.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import RaceDataset
+
+race_reader_cfg = dict(
+    input_columns=['article', 'question', 'A', 'B', 'C', 'D'],
+    output_column='answer')
+
+race_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role="HUMAN",
+                prompt=
+                "Read the article, and answer the question by replying A, B, C or D.\n\nArticle:\n{article}\n\nQ: {question}\n\nA. {A}\nB. {B}\nC. {C}\nD. {D}"
+            ),
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+race_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_postprocessor=dict(type='first-capital'),
+    pred_role='BOT')
+
+race_datasets = [
+    dict(
+        type=RaceDataset,
+        abbr='race-middle',
+        path='race',
+        name='middle',
+        reader_cfg=race_reader_cfg,
+        infer_cfg=race_infer_cfg,
+        eval_cfg=race_eval_cfg),
+    dict(
+        type=RaceDataset,
+        abbr='race-high',
+        path='race',
+        name='high',
+        reader_cfg=race_reader_cfg,
+        infer_cfg=race_infer_cfg,
+        eval_cfg=race_eval_cfg)
+]

configs/datasets/race/race_gen_d18b89.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import RaceDataset
+
+race_reader_cfg = dict(
+    input_columns=['article', 'question', 'A', 'B', 'C', 'D'],
+    output_column='answer')
+
+race_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=
+        'Read the article, and answer the question by replying A, B, C or D.\n\n{article}\n\nQ: {question}\n\nA. {A}\nB. {B}\nC. {C}\nD. {D}'),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+race_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_postprocessor=dict(type='first-capital'))
+
+race_datasets = [
+    dict(
+        type=RaceDataset,
+        abbr='race-middle',
+        path='race',
+        name='middle',
+        reader_cfg=race_reader_cfg,
+        infer_cfg=race_infer_cfg,
+        eval_cfg=race_eval_cfg),
+    dict(
+        type=RaceDataset,
+        abbr='race-high',
+        path='race',
+        name='high',
+        reader_cfg=race_reader_cfg,
+        infer_cfg=race_infer_cfg,
+        eval_cfg=race_eval_cfg)
+]

configs/datasets/race/race_ppl.py

+from mmengine.config import read_base
+
+with read_base():
+    from .race_ppl_04e06a import race_datasets  # noqa: F401, F403

configs/datasets/realtoxicprompts/realtoxicprompts_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .realtoxicprompts_gen_3ea730 import realtoxicprompts_datasets  # noqa: F401, F403

configs/datasets/safety/safety_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .safety_gen_c0a5b8 import safety_datasets  # noqa: F401, F403

configs/datasets/siqa/siqa_gen_a3c714.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import siqaDataset_V2
+
+siqa_reader_cfg = dict(
+    input_columns=["context", "question", "answerA", "answerB", "answerC"],
+    output_column="label",
+    test_split="validation")
+
+siqa_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role="HUMAN",
+                    prompt=
+                    "{context}\nQuestion: {question}\nA. {answerA}\nB. {answerB}\nC. {answerC}\nAnswer:"
+                )
+            ], ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+siqa_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role="BOT",
+    pred_postprocessor=dict(type="first-capital"),
+)
+
+siqa_datasets = [
+    dict(
+        abbr="siqa",
+        type=siqaDataset_V2,
+        path="social_i_qa",
+        reader_cfg=siqa_reader_cfg,
+        infer_cfg=siqa_infer_cfg,
+        eval_cfg=siqa_eval_cfg)
+]

configs/datasets/siqa/siqa_ppl.py

+from mmengine.config import read_base
+
+with read_base():
+    from .siqa_ppl_049da0 import siqa_datasets  # noqa: F401, F403

configs/datasets/storycloze/storycloze_ppl.py

+from mmengine.config import read_base
+
+with read_base():
+    from .storycloze_ppl_c1912d import storycloze_datasets  # noqa: F401, F403

configs/datasets/storycloze/storycloze_ppl_7f4c64.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import storyclozeDataset
+
+storycloze_reader_cfg = dict(
+    input_columns=['context', 'sentence_quiz1', 'sentence_quiz2'],
+    output_column='answer_right_ending',
+    train_split='test',
+    test_split='test')
+
+storycloze_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            1: "{context}{sentence_quiz1}",
+            2: "{context}{sentence_quiz2}",
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+storycloze_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+# The original story cloze dataset and repo are not long maintaining.
+# Using multilingual version of this dataset.
+storycloze_datasets = [
+    dict(
+        abbr='story_cloze',
+        type=storyclozeDataset,
+        path='juletxara/xstory_cloze',
+        name='en',
+        reader_cfg=storycloze_reader_cfg,
+        infer_cfg=storycloze_infer_cfg,
+        eval_cfg=storycloze_eval_cfg)
+]

configs/datasets/summedits/summedits_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .summedits_gen_4f35b5 import summedits_datasets  # noqa: F401, F403

configs/datasets/summedits/summedits_gen_4f35b5.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import SummeditsDataset_V2
+
+summedits_reader_cfg = dict(
+    input_columns=['doc', 'summary'], output_column='label')
+
+summedits_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role="HUMAN",
+                prompt=
+                'Document:\n{doc}Summary:\n{summary}\nQuestion:\nIs the summary factually consistent with the document?\nA. Yes\nB. No\nAnswer:'
+            ),
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+summedits_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role="BOT",
+    pred_postprocessor=dict(type="first-capital"),
+)
+
+summedits_datasets = [
+    dict(
+        abbr='summedits',
+        type=SummeditsDataset_V2,
+        path='./data/summedits/summedits.jsonl',
+        reader_cfg=summedits_reader_cfg,
+        infer_cfg=summedits_infer_cfg,
+        eval_cfg=summedits_eval_cfg)
+]

configs/datasets/triviaqarc/triviaqarc_gen_a02306.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import TriviaQArcDataset, TriviaQAEvaluator
+
+triviaqarc_reader_cfg = dict(
+    input_columns=['question', 'evidence'],
+    output_column='answer',
+    train_split='dev',
+    test_split='dev')
+
+triviaqarc_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template="{evidence}\nAnswer these questions:\nQ: {question}?\nA:"),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(
+        type=GenInferencer, max_out_len=50, max_seq_len=8192, batch_size=4))
+
+triviaqarc_eval_cfg = dict(evaluator=dict(type=TriviaQAEvaluator))
+
+triviaqarc_datasets = [
+    dict(
+        type=TriviaQArcDataset,
+        abbr='triviaqarc',
+        path='./data/triviaqa-rc/',
+        reader_cfg=triviaqarc_reader_cfg,
+        infer_cfg=triviaqarc_infer_cfg,
+        eval_cfg=triviaqarc_eval_cfg)
+]

configs/models/classic/tigerbot-7b-sft.py

+from opencompass.models import HuggingFaceCausalLM
+
+_meta_template = dict(
+    round=[
+        dict(role='HUMAN', begin='\n\n### Instruction:\n:'), 
+        dict(role='BOT', begin='\n\n### Response:\n:', generate=True),
+    ],
+)
+
+
+models = [
+    dict(
+        type=HuggingFaceCausalLM,
+        abbr='TigerBot-SFT',
+        path="TigerResearch/tigerbot-7b-sft",
+        tokenizer_path='TigerResearch/tigerbot-7b-sft',
+        tokenizer_kwargs=dict(
+            padding_side='left',
+            truncation_side='left',
+            trust_remote_code=True,
+        ),
+        max_out_len=100,
+        max_seq_len=2048,
+        batch_size=8,
+        meta_template=_meta_template,
+        model_kwargs=dict(trust_remote_code=True, device_map='auto', revision='0ba4d6fc479bdedd6a3f8d4d3425025c5f501800'),
+        run_cfg=dict(num_gpus=1, num_procs=1),
+    )
+]

configs/summarizers/groups/bbh.py

+bbh_summary_groups = []
+
+# bbh
+_bbh = ['temporal_sequences', 'disambiguation_qa', 'date_understanding', 'tracking_shuffled_objects_three_objects', 'penguins_in_a_table','geometric_shapes', 'snarks', 'ruin_names', 'tracking_shuffled_objects_seven_objects', 'tracking_shuffled_objects_five_objects','logical_deduction_three_objects', 'hyperbaton', 'logical_deduction_five_objects', 'logical_deduction_seven_objects', 'movie_recommendation','salient_translation_error_detection', 'reasoning_about_colored_objects', 'multistep_arithmetic_two', 'navigate', 'dyck_languages', 'word_sorting', 'sports_understanding','boolean_expressions', 'object_counting', 'formal_fallacies', 'causal_judgement', 'web_of_lies']
+_bbh = ['bbh-' + s for s in _bbh]
+bbh_summary_groups.append({'name': 'bbh', 'subsets': _bbh})

docs/en/_templates/404.html

+{% extends "layout.html" %}
+
+{% block body %}
+
+<h1>Page Not Found</h1>
+<p>
+  The page you are looking for cannot be found.
+</p>
+<p>
+  If you just switched documentation versions, it is likely that the page you were on is moved. You can look for it in
+  the content table left, or go to <a href="{{ pathto(root_doc) }}">the homepage</a>.
+</p>
+<!-- <p>
+  If you cannot find documentation you want, please <a
+    href="">open an issue</a> to tell us!
+</p> -->
+
+{% endblock %}