wikitext_2_raw_ppl_752e2a.py

from opencompass.openicl.icl_prompt_template import PromptTemplate
from opencompass.openicl.icl_retriever import ZeroRetriever
from opencompass.openicl.icl_inferencer import GenInferencer
from opencompass.openicl.icl_evaluator import AccEvaluator
from opencompass.datasets import HFDataset


wikitext_infer_cfg = dict(
    prompt_template=dict(
        type=PromptTemplate,
        template={0: '{text}'}
    ),
    retriever=dict(type=ZeroRetriever),
    inferencer=dict(type=GenInferencer), # need a new ppl inferencer
)

wikitext_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )

wikitext_2_raw_datasets = []
for _split in ['validation', 'test']:

    wikitext_reader_cfg = dict(
        input_columns=['text'],
        output_column=None,
        train_split='train',
        test_split=_split,
    )

    wikitext_2_raw_datasets.append(
        dict(
            abbr=f'wikitext-2-raw-{_split}',
            type=HFDataset,
            path='wikitext',
            name='wikitext-2-raw-v1',
            reader_cfg=wikitext_reader_cfg,
            infer_cfg=wikitext_infer_cfg,
            eval_cfg=wikitext_eval_cfg,
        )
    )