Skip to content

GitLab

"...fattn-vec-f16-instance-hs128-q5_0-q5_0.cu" did not exist on "de982616f1dde636e46b2cef2edd971b54ef7691"
Switch branch/tag
Find file Normal viewHistoryPermalink
_mmlu.yaml 867 Bytes
Newer Older
Luis Cosio's avatar
feat: Add mmlu-redux and it's spanish transaltion as generative task definitions (#2705)  
Luis Cosio committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33 
group: mmlu_redux_spanish_generative
group_alias: mmlu_redux_spanish (generative)
task:
  - group: stem_spanish
    task:
      - mmlu_stem_generative_spanish
    aggregate_metric_list:
      - metric: exact_match
        weight_by_size: true
  - group: other_spanish
    task:
      - mmlu_other_generative_spanish
    aggregate_metric_list:
      - metric: exact_match
        weight_by_size: true
  - group: social sciences_spanish
    task:
      - mmlu_social_sciences_generative_spanish
    aggregate_metric_list:
      - metric: exact_match
        weight_by_size: true
#  - group: humanities_spanish
#    task:
#      - mmlu_humanities_generative_spanish
#    aggregate_metric_list:
#      - metric: exact_match
#        weight_by_size: true
aggregate_metric_list:
  - aggregation: mean
    metric: exact_match
    weight_by_size: true
metadata:
  version: 3