Skip to content

GitLab

Switch branch/tag
Find file Normal viewHistoryPermalink
eval_codeagent.py 1.61 KB
Newer Older
jerrrrry's avatar
Initial commit  
jerrrrry committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52 
from mmengine.config import read_base

from opencompass.models import HuggingFaceCausalLM, OpenAI
from opencompass.models.lagent import CodeAgent
from opencompass.partitioners import SizePartitioner
from opencompass.runners import LocalRunner
from opencompass.tasks import OpenICLInferTask

with read_base():
    from opencompass.configs.datasets.gsm8k.gsm8k_gen_57b0b1 import \
        gsm8k_datasets
    from opencompass.configs.datasets.math.math_gen_943d32 import math_datasets

datasets = []
datasets += gsm8k_datasets
datasets += math_datasets

models = [
    dict(abbr='gpt-3.5-react',
         type=CodeAgent,
         llm=dict(
             type=OpenAI,
             path='gpt-3.5-turbo',
             key='ENV',
             query_per_second=1,
             max_seq_len=4096,
         ),
         batch_size=8),
    dict(abbr='WizardCoder-Python-13B-V1.0-react',
         type=CodeAgent,
         llm=dict(
             type=HuggingFaceCausalLM,
             path='WizardLM/WizardCoder-Python-13B-V1.0',
             tokenizer_path='WizardLM/WizardCoder-Python-13B-V1.0',
             tokenizer_kwargs=dict(
                 padding_side='left',
                 truncation_side='left',
                 trust_remote_code=True,
             ),
             max_seq_len=2048,
             model_kwargs=dict(trust_remote_code=True, device_map='auto'),
         ),
         batch_size=8,
         run_cfg=dict(num_gpus=2, num_procs=1)),
]

infer = dict(
    partitioner=dict(type=SizePartitioner, max_task_size=40000),
    runner=dict(type=LocalRunner,
                max_num_workers=16,
                task=dict(type=OpenICLInferTask)),
)