winogrande_5shot_ll_252f01.py 1.24 KB
Newer Older
Fengzhe Zhou's avatar
Fengzhe Zhou committed
1
2
3
from opencompass.openicl.icl_prompt_template import PromptTemplate
from opencompass.openicl.icl_retriever import FixKRetriever
from opencompass.openicl.icl_inferencer import LLInferencer
Fengzhe Zhou's avatar
Fengzhe Zhou committed
4
from opencompass.openicl.icl_evaluator import AccwithDetailsEvaluator
Fengzhe Zhou's avatar
Fengzhe Zhou committed
5
6
7
8
9
from opencompass.datasets import winograndeDataset_V3

winogrande_reader_cfg = dict(
    input_columns=['opt1', 'opt2'],
    output_column='answer',
10
11
    train_split='train_xs',
    test_split='dev',
Fengzhe Zhou's avatar
Fengzhe Zhou committed
12
13
)

14
question_and_options = 'Which of the following is a good sentence:\nA. {opt1}\nB. {opt2}'
Fengzhe Zhou's avatar
Fengzhe Zhou committed
15
16
17
winogrande_infer_cfg = dict(
    ice_template=dict(
        type=PromptTemplate,
Fengzhe Zhou's avatar
Fengzhe Zhou committed
18
        template={'A': '{opt1}', 'B': '{opt2}'},
Fengzhe Zhou's avatar
Fengzhe Zhou committed
19
20
21
    ),
    prompt_template=dict(
        type=PromptTemplate,
Fengzhe Zhou's avatar
Fengzhe Zhou committed
22
        template={'A': '</E>{opt1}', 'B': '</E>{opt2}'},
23
        ice_token='</E>',
Fengzhe Zhou's avatar
Fengzhe Zhou committed
24
25
26
27
    ),
    retriever=dict(type=FixKRetriever, fix_id_list=[0, 2, 4, 6, 8]),
    inferencer=dict(type=LLInferencer),
)
Fengzhe Zhou's avatar
Fengzhe Zhou committed
28
winogrande_eval_cfg = dict(evaluator=dict(type=AccwithDetailsEvaluator))
Fengzhe Zhou's avatar
Fengzhe Zhou committed
29
30
31
32
33
34
35
36
37
38

winogrande_datasets = [
    dict(
        abbr='winogrande',
        type=winograndeDataset_V3,
        path='./data/winogrande',
        reader_cfg=winogrande_reader_cfg,
        infer_cfg=winogrande_infer_cfg,
        eval_cfg=winogrande_eval_cfg)
]