Skip to content

GitLab

Switch branch/tag
Find file Normal viewHistoryPermalink
NPHardEval_gen_22aac5.py 2.14 KB
Newer Older
Skyfall-xzz's avatar
Support NPHardEval (#835)  
Skyfall-xzz committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59 
from opencompass.openicl.icl_prompt_template import PromptTemplate
from opencompass.openicl.icl_retriever import ZeroRetriever
from opencompass.openicl.icl_inferencer import GenInferencer
from opencompass.datasets.NPHardEval import (
    hard_GCP_Dataset, hard_GCP_Evaluator,
    hard_TSP_Dataset, hard_TSP_Evaluator,
    hard_MSP_Dataset, hard_MSP_Evaluator,
    cmp_GCP_D_Dataset, cmp_GCP_D_Evaluator,
    cmp_TSP_D_Dataset, cmp_TSP_D_Evaluator,
    cmp_KSP_Dataset, cmp_KSP_Evaluator,
    p_BSP_Dataset, p_BSP_Evaluator,
    p_EDP_Dataset, p_EDP_Evaluator,
    p_SPP_Dataset, p_SPP_Evaluator,
)

NPHardEval_tasks = [
    ["hard_GCP", "GCP", hard_GCP_Dataset, hard_GCP_Evaluator],
    ["hard_TSP", "TSP", hard_TSP_Dataset, hard_TSP_Evaluator],
    ["hard_MSP", "MSP", hard_MSP_Dataset, hard_MSP_Evaluator],
    ["cmp_GCP_D", "GCP_Decision", cmp_GCP_D_Dataset, cmp_GCP_D_Evaluator],
    ["cmp_TSP_D", "TSP_Decision", cmp_TSP_D_Dataset, cmp_TSP_D_Evaluator],
    ["cmp_KSP", "KSP", cmp_KSP_Dataset, cmp_KSP_Evaluator],
    ["p_BSP", "BSP", p_BSP_Dataset, p_BSP_Evaluator],
    ["p_EDP", "EDP", p_EDP_Dataset, p_EDP_Evaluator],
    ["p_SPP", "SPP", p_SPP_Dataset, p_SPP_Evaluator],
]

NPHardEval_datasets = []
for name, path_name, dataset, evaluator in NPHardEval_tasks:
    NPHardEval_reader_cfg = dict(input_columns=["prompt", "level"], output_column="q")

    NPHardEval_infer_cfg = dict(
        ice_template=dict(
            type=PromptTemplate,
            template=dict(
                begin="</E>",
                round=[
                    dict(role="HUMAN", prompt="</E>{prompt}"),
                    dict(role="BOT", prompt=""),
                ],
            ),
            ice_token="</E>",
        ),
        retriever=dict(type=ZeroRetriever),
        inferencer=dict(type=GenInferencer),
    )

    NPHardEval_eval_cfg = dict(evaluator=dict(type=evaluator), pred_role="BOT")

    NPHardEval_datasets.append(
        dict(
            type=dataset,
            abbr=name,
            path=f"./data/NPHardEval/{path_name}/",
            reader_cfg=NPHardEval_reader_cfg,
            infer_cfg=NPHardEval_infer_cfg,
            eval_cfg=NPHardEval_eval_cfg,
        )
    )