Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
gaoqiong
lm-evaluation-harness
Commits
2e8aa9c2
Commit
2e8aa9c2
authored
Oct 14, 2025
by
Baber
Browse files
update mrl subsets
parent
d94921d6
Changes
40
Show whitespace changes
Inline
Side-by-side
Showing
20 changed files
with
95 additions
and
0 deletions
+95
-0
lm_eval/tasks/mrl/mc/_global_piqa.yaml
lm_eval/tasks/mrl/mc/_global_piqa.yaml
+38
-0
lm_eval/tasks/mrl/mc/acm_arab.yaml
lm_eval/tasks/mrl/mc/acm_arab.yaml
+3
-0
lm_eval/tasks/mrl/mc/asm_beng.yaml
lm_eval/tasks/mrl/mc/asm_beng.yaml
+3
-0
lm_eval/tasks/mrl/mc/ben_beng.yaml
lm_eval/tasks/mrl/mc/ben_beng.yaml
+3
-0
lm_eval/tasks/mrl/mc/bos_latn.yaml
lm_eval/tasks/mrl/mc/bos_latn.yaml
+3
-0
lm_eval/tasks/mrl/mc/deu_latn.yaml
lm_eval/tasks/mrl/mc/deu_latn.yaml
+3
-0
lm_eval/tasks/mrl/mc/ekp_latn.yaml
lm_eval/tasks/mrl/mc/ekp_latn.yaml
+3
-0
lm_eval/tasks/mrl/mc/hin_deva.yaml
lm_eval/tasks/mrl/mc/hin_deva.yaml
+3
-0
lm_eval/tasks/mrl/mc/kan_knda.yaml
lm_eval/tasks/mrl/mc/kan_knda.yaml
+3
-0
lm_eval/tasks/mrl/mc/kin_latn.yaml
lm_eval/tasks/mrl/mc/kin_latn.yaml
+3
-0
lm_eval/tasks/mrl/mc/lin_latn.yaml
lm_eval/tasks/mrl/mc/lin_latn.yaml
+3
-0
lm_eval/tasks/mrl/mc/luo_latn.yaml
lm_eval/tasks/mrl/mc/luo_latn.yaml
+3
-0
lm_eval/tasks/mrl/mc/mal_mlym.yaml
lm_eval/tasks/mrl/mc/mal_mlym.yaml
+3
-0
lm_eval/tasks/mrl/mc/mni_mtei.yaml
lm_eval/tasks/mrl/mc/mni_mtei.yaml
+3
-0
lm_eval/tasks/mrl/mc/pan_guru.yaml
lm_eval/tasks/mrl/mc/pan_guru.yaml
+3
-0
lm_eval/tasks/mrl/mc/rus_cyrl.yaml
lm_eval/tasks/mrl/mc/rus_cyrl.yaml
+3
-0
lm_eval/tasks/mrl/mc/spa_latn_mexi.yaml
lm_eval/tasks/mrl/mc/spa_latn_mexi.yaml
+3
-0
lm_eval/tasks/mrl/mc/spa_latn_spai.yaml
lm_eval/tasks/mrl/mc/spa_latn_spai.yaml
+3
-0
lm_eval/tasks/mrl/mc/swh_latn.yaml
lm_eval/tasks/mrl/mc/swh_latn.yaml
+3
-0
lm_eval/tasks/mrl/mc/tel_telu.yaml
lm_eval/tasks/mrl/mc/tel_telu.yaml
+3
-0
No files found.
lm_eval/tasks/mrl/mc/_global_piqa.yaml
View file @
2e8aa9c2
group
:
global_piqa
group
:
global_piqa
task
:
task
:
-
task
:
mrl_acm_arab
task_alias
:
acm_arab
-
task
:
mrl_acq_arab
-
task
:
mrl_acq_arab
task_alias
:
acq_arab
task_alias
:
acq_arab
-
task
:
mrl_aeb_arab
-
task
:
mrl_aeb_arab
...
@@ -28,16 +30,22 @@ task:
...
@@ -28,16 +30,22 @@ task:
task_alias
:
ary_arab
task_alias
:
ary_arab
-
task
:
mrl_arz_arab
-
task
:
mrl_arz_arab
task_alias
:
arz_arab
task_alias
:
arz_arab
-
task
:
mrl_asm_beng
task_alias
:
asm_beng
-
task
:
mrl_azj_latn
-
task
:
mrl_azj_latn
task_alias
:
azj_latn
task_alias
:
azj_latn
-
task
:
mrl_bam_latn
-
task
:
mrl_bam_latn
task_alias
:
bam_latn
task_alias
:
bam_latn
-
task
:
mrl_bel_cyrl
-
task
:
mrl_bel_cyrl
task_alias
:
bel_cyrl
task_alias
:
bel_cyrl
-
task
:
mrl_ben_beng
task_alias
:
ben_beng
-
task
:
mrl_ben_latn
-
task
:
mrl_ben_latn
task_alias
:
ben_latn
task_alias
:
ben_latn
-
task
:
mrl_bho_deva
-
task
:
mrl_bho_deva
task_alias
:
bho_deva
task_alias
:
bho_deva
-
task
:
mrl_bos_latn
task_alias
:
bos_latn
-
task
:
mrl_bsk_arab
-
task
:
mrl_bsk_arab
task_alias
:
bsk_arab
task_alias
:
bsk_arab
-
task
:
mrl_bul_cyrl
-
task
:
mrl_bul_cyrl
...
@@ -54,8 +62,12 @@ task:
...
@@ -54,8 +62,12 @@ task:
task_alias
:
cmn_hans
task_alias
:
cmn_hans
-
task
:
mrl_cmn_hant
-
task
:
mrl_cmn_hant
task_alias
:
cmn_hant
task_alias
:
cmn_hant
-
task
:
mrl_deu_latn
task_alias
:
deu_latn
-
task
:
mrl_dhd_deva
-
task
:
mrl_dhd_deva
task_alias
:
dhd_deva
task_alias
:
dhd_deva
-
task
:
mrl_ekp_latn
task_alias
:
ekp_latn
-
task
:
mrl_ell_grek
-
task
:
mrl_ell_grek
task_alias
:
ell_grek
task_alias
:
ell_grek
-
task
:
mrl_eng_latn
-
task
:
mrl_eng_latn
...
@@ -80,6 +92,8 @@ task:
...
@@ -80,6 +92,8 @@ task:
task_alias
:
haw_latn
task_alias
:
haw_latn
-
task
:
mrl_heb_hebr
-
task
:
mrl_heb_hebr
task_alias
:
heb_hebr
task_alias
:
heb_hebr
-
task
:
mrl_hin_deva
task_alias
:
hin_deva
-
task
:
mrl_hrv_latn
-
task
:
mrl_hrv_latn
task_alias
:
hrv_latn
task_alias
:
hrv_latn
-
task
:
mrl_hun_latn
-
task
:
mrl_hun_latn
...
@@ -102,22 +116,34 @@ task:
...
@@ -102,22 +116,34 @@ task:
task_alias
:
jav_latn
task_alias
:
jav_latn
-
task
:
mrl_jpn_jpan
-
task
:
mrl_jpn_jpan
task_alias
:
jpn_jpan
task_alias
:
jpn_jpan
-
task
:
mrl_kan_knda
task_alias
:
kan_knda
-
task
:
mrl_kat_geor
-
task
:
mrl_kat_geor
task_alias
:
kat_geor
task_alias
:
kat_geor
-
task
:
mrl_kaz_cyrl
-
task
:
mrl_kaz_cyrl
task_alias
:
kaz_cyrl
task_alias
:
kaz_cyrl
-
task
:
mrl_kin_latn
task_alias
:
kin_latn
-
task
:
mrl_kir_cyrl
-
task
:
mrl_kir_cyrl
task_alias
:
kir_cyrl
task_alias
:
kir_cyrl
-
task
:
mrl_kor_hang
-
task
:
mrl_kor_hang
task_alias
:
kor_hang
task_alias
:
kor_hang
-
task
:
mrl_lin_latn
task_alias
:
lin_latn
-
task
:
mrl_lit_latn
-
task
:
mrl_lit_latn
task_alias
:
lit_latn
task_alias
:
lit_latn
-
task
:
mrl_luo_latn
task_alias
:
luo_latn
-
task
:
mrl_mal_mlym
task_alias
:
mal_mlym
-
task
:
mrl_mar_deva
-
task
:
mrl_mar_deva
task_alias
:
mar_deva
task_alias
:
mar_deva
-
task
:
mrl_mkd_cyrl
-
task
:
mrl_mkd_cyrl
task_alias
:
mkd_cyrl
task_alias
:
mkd_cyrl
-
task
:
mrl_mni_beng
-
task
:
mrl_mni_beng
task_alias
:
mni_beng
task_alias
:
mni_beng
-
task
:
mrl_mni_mtei
task_alias
:
mni_mtei
-
task
:
mrl_nag_latn
-
task
:
mrl_nag_latn
task_alias
:
nag_latn
task_alias
:
nag_latn
-
task
:
mrl_nld_latn
-
task
:
mrl_nld_latn
...
@@ -128,6 +154,8 @@ task:
...
@@ -128,6 +154,8 @@ task:
task_alias
:
nob_latn
task_alias
:
nob_latn
-
task
:
mrl_npi_deva
-
task
:
mrl_npi_deva
task_alias
:
npi_deva
task_alias
:
npi_deva
-
task
:
mrl_pan_guru
task_alias
:
pan_guru
-
task
:
mrl_pcm_latn
-
task
:
mrl_pcm_latn
task_alias
:
pcm_latn
task_alias
:
pcm_latn
-
task
:
mrl_pes_arab
-
task
:
mrl_pes_arab
...
@@ -140,6 +168,8 @@ task:
...
@@ -140,6 +168,8 @@ task:
task_alias
:
por_latn_port
task_alias
:
por_latn_port
-
task
:
mrl_ron_latn
-
task
:
mrl_ron_latn
task_alias
:
ron_latn
task_alias
:
ron_latn
-
task
:
mrl_rus_cyrl
task_alias
:
rus_cyrl
-
task
:
mrl_rwr_deva
-
task
:
mrl_rwr_deva
task_alias
:
rwr_deva
task_alias
:
rwr_deva
-
task
:
mrl_sin_sinh
-
task
:
mrl_sin_sinh
...
@@ -156,16 +186,24 @@ task:
...
@@ -156,16 +186,24 @@ task:
task_alias
:
snd_arab
task_alias
:
snd_arab
-
task
:
mrl_snd_deva
-
task
:
mrl_snd_deva
task_alias
:
snd_deva
task_alias
:
snd_deva
-
task
:
mrl_spa_latn_mexi
task_alias
:
spa_latn_mexi
-
task
:
mrl_spa_latn_peru
-
task
:
mrl_spa_latn_peru
task_alias
:
spa_latn_peru
task_alias
:
spa_latn_peru
-
task
:
mrl_spa_latn_spai
task_alias
:
spa_latn_spai
-
task
:
mrl_srp_cyrl
-
task
:
mrl_srp_cyrl
task_alias
:
srp_cyrl
task_alias
:
srp_cyrl
-
task
:
mrl_srp_latn
-
task
:
mrl_srp_latn
task_alias
:
srp_latn
task_alias
:
srp_latn
-
task
:
mrl_swe_latn
-
task
:
mrl_swe_latn
task_alias
:
swe_latn
task_alias
:
swe_latn
-
task
:
mrl_swh_latn
task_alias
:
swh_latn
-
task
:
mrl_tam_taml
-
task
:
mrl_tam_taml
task_alias
:
tam_taml
task_alias
:
tam_taml
-
task
:
mrl_tel_telu
task_alias
:
tel_telu
-
task
:
mrl_tgl_latn
-
task
:
mrl_tgl_latn
task_alias
:
tgl_latn
task_alias
:
tgl_latn
-
task
:
mrl_tha_thai
-
task
:
mrl_tha_thai
...
...
lm_eval/tasks/mrl/mc/acm_arab.yaml
0 → 100644
View file @
2e8aa9c2
include
:
'
_template_mc'
task
:
mrl_acm_arab
dataset_name
:
acm_arab
lm_eval/tasks/mrl/mc/asm_beng.yaml
0 → 100644
View file @
2e8aa9c2
include
:
'
_template_mc'
task
:
mrl_asm_beng
dataset_name
:
asm_beng
lm_eval/tasks/mrl/mc/ben_beng.yaml
0 → 100644
View file @
2e8aa9c2
include
:
'
_template_mc'
task
:
mrl_ben_beng
dataset_name
:
ben_beng
lm_eval/tasks/mrl/mc/bos_latn.yaml
0 → 100644
View file @
2e8aa9c2
include
:
'
_template_mc'
task
:
mrl_bos_latn
dataset_name
:
bos_latn
lm_eval/tasks/mrl/mc/deu_latn.yaml
0 → 100644
View file @
2e8aa9c2
include
:
'
_template_mc'
task
:
mrl_deu_latn
dataset_name
:
deu_latn
lm_eval/tasks/mrl/mc/ekp_latn.yaml
0 → 100644
View file @
2e8aa9c2
include
:
'
_template_mc'
task
:
mrl_ekp_latn
dataset_name
:
ekp_latn
lm_eval/tasks/mrl/mc/hin_deva.yaml
0 → 100644
View file @
2e8aa9c2
include
:
'
_template_mc'
task
:
mrl_hin_deva
dataset_name
:
hin_deva
lm_eval/tasks/mrl/mc/kan_knda.yaml
0 → 100644
View file @
2e8aa9c2
include
:
'
_template_mc'
task
:
mrl_kan_knda
dataset_name
:
kan_knda
lm_eval/tasks/mrl/mc/kin_latn.yaml
0 → 100644
View file @
2e8aa9c2
include
:
'
_template_mc'
task
:
mrl_kin_latn
dataset_name
:
kin_latn
lm_eval/tasks/mrl/mc/lin_latn.yaml
0 → 100644
View file @
2e8aa9c2
include
:
'
_template_mc'
task
:
mrl_lin_latn
dataset_name
:
lin_latn
lm_eval/tasks/mrl/mc/luo_latn.yaml
0 → 100644
View file @
2e8aa9c2
include
:
'
_template_mc'
task
:
mrl_luo_latn
dataset_name
:
luo_latn
lm_eval/tasks/mrl/mc/mal_mlym.yaml
0 → 100644
View file @
2e8aa9c2
include
:
'
_template_mc'
task
:
mrl_mal_mlym
dataset_name
:
mal_mlym
lm_eval/tasks/mrl/mc/mni_mtei.yaml
0 → 100644
View file @
2e8aa9c2
include
:
'
_template_mc'
task
:
mrl_mni_mtei
dataset_name
:
mni_mtei
lm_eval/tasks/mrl/mc/pan_guru.yaml
0 → 100644
View file @
2e8aa9c2
include
:
'
_template_mc'
task
:
mrl_pan_guru
dataset_name
:
pan_guru
lm_eval/tasks/mrl/mc/rus_cyrl.yaml
0 → 100644
View file @
2e8aa9c2
include
:
'
_template_mc'
task
:
mrl_rus_cyrl
dataset_name
:
rus_cyrl
lm_eval/tasks/mrl/mc/spa_latn_mexi.yaml
0 → 100644
View file @
2e8aa9c2
include
:
'
_template_mc'
task
:
mrl_spa_latn_mexi
dataset_name
:
spa_latn_mexi
lm_eval/tasks/mrl/mc/spa_latn_spai.yaml
0 → 100644
View file @
2e8aa9c2
include
:
'
_template_mc'
task
:
mrl_spa_latn_spai
dataset_name
:
spa_latn_spai
lm_eval/tasks/mrl/mc/swh_latn.yaml
0 → 100644
View file @
2e8aa9c2
include
:
'
_template_mc'
task
:
mrl_swh_latn
dataset_name
:
swh_latn
lm_eval/tasks/mrl/mc/tel_telu.yaml
0 → 100644
View file @
2e8aa9c2
include
:
'
_template_mc'
task
:
mrl_tel_telu
dataset_name
:
tel_telu
Prev
1
2
Next
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment