Skip to content

GitLab

"lm_eval/tasks/umtceval/default/umtceval.yaml" did not exist on "3c0df6e9640e727c1d8948c45e3c2de33d1f1e35"
Switch branch/tag
Find file Normal viewHistoryPermalink
flan_held_in.yaml 1.03 KB
Newer Older
lintangsutawika's avatar
expanded benchmark to allow new source of prompt templates  
lintangsutawika committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37 
group: flan_held_in
task:
  - include: held_in_template_yaml
    dataset_path: super_glue
    dataset_name: boolq
    use_prompt: flan_boolq.yaml:*
    validation_split: validation
  # - include: held_in_template_yaml
  #   dataset_path: super_glue
  #   dataset_name: rte
  #   use_prompt: local:*
  #   validation_split: validation
  # - include: held_in_template_yaml
  #   task: anli_r1
  #   dataset_path: anli
  #   use_prompt: local:*
  #   validation_split: dev_r1
  # - include: held_in_template_yaml
  #   task: anli_r2
  #   dataset_path: anli
  #   use_prompt: local:*
  #   validation_split: dev_r2
  # - include: held_in_template_yaml
  #   task: anli_r3
  #   dataset_path: anli
  #   use_prompt: local:*
  #   validation_split: dev_r3
  # - include: held_in_template_yaml
  #   task: ai2_arc
  #   dataset_path: ARC-Easy
  #   use_prompt: local:*
  #   validation_split: validation
  # - include: held_in_template_yaml
  #   task: ai2_arc
  #   dataset_path: ARC-Challange
  #   use_prompt: local:*
  #   validation_split: validation