Add the MultiBLiMP benchmark (#3155)

* add multiblimp * run linter

Add the MultiBLiMP benchmark (#3155)
* add multiblimp * run linter
de4ce482 · James A. Michaelov · GitHub · 8c025369 · de4ce482 · de4ce482
Unverified Commit de4ce482 authored Jul 19, 2025 by James A. Michaelov Committed by GitHub Jul 19, 2025
20 changed files
--- a/lm_eval/tasks/multiblimp/multiblimp_lij.yaml
+++ b/lm_eval/tasks/multiblimp/multiblimp_lij.yaml
+include: _template_yaml
+dataset_name: lij
+task: multiblimp_lij
--- a/lm_eval/tasks/multiblimp/multiblimp_lit.yaml
+++ b/lm_eval/tasks/multiblimp/multiblimp_lit.yaml
+include: _template_yaml
+dataset_name: lit
+task: multiblimp_lit
--- a/lm_eval/tasks/multiblimp/multiblimp_mar.yaml
+++ b/lm_eval/tasks/multiblimp/multiblimp_mar.yaml
+include: _template_yaml
+dataset_name: mar
+task: multiblimp_mar
--- a/lm_eval/tasks/multiblimp/multiblimp_mdf.yaml
+++ b/lm_eval/tasks/multiblimp/multiblimp_mdf.yaml
+include: _template_yaml
+dataset_name: mdf
+task: multiblimp_mdf
--- a/lm_eval/tasks/multiblimp/multiblimp_mkd.yaml
+++ b/lm_eval/tasks/multiblimp/multiblimp_mkd.yaml
+include: _template_yaml
+dataset_name: mkd
+task: multiblimp_mkd
--- a/lm_eval/tasks/multiblimp/multiblimp_myv.yaml
+++ b/lm_eval/tasks/multiblimp/multiblimp_myv.yaml
+include: _template_yaml
+dataset_name: myv
+task: multiblimp_myv
--- a/lm_eval/tasks/multiblimp/multiblimp_nds.yaml
+++ b/lm_eval/tasks/multiblimp/multiblimp_nds.yaml
+include: _template_yaml
+dataset_name: nds
+task: multiblimp_nds
--- a/lm_eval/tasks/multiblimp/multiblimp_nhi.yaml
+++ b/lm_eval/tasks/multiblimp/multiblimp_nhi.yaml
+include: _template_yaml
+dataset_name: nhi
+task: multiblimp_nhi
--- a/lm_eval/tasks/multiblimp/multiblimp_nld.yaml
+++ b/lm_eval/tasks/multiblimp/multiblimp_nld.yaml
+include: _template_yaml
+dataset_name: nld
+task: multiblimp_nld
--- a/lm_eval/tasks/multiblimp/multiblimp_olo.yaml
+++ b/lm_eval/tasks/multiblimp/multiblimp_olo.yaml
+include: _template_yaml
+dataset_name: olo
+task: multiblimp_olo
--- a/lm_eval/tasks/multiblimp/multiblimp_orv.yaml
+++ b/lm_eval/tasks/multiblimp/multiblimp_orv.yaml
+include: _template_yaml
+dataset_name: orv
+task: multiblimp_orv
--- a/lm_eval/tasks/multiblimp/multiblimp_ota.yaml
+++ b/lm_eval/tasks/multiblimp/multiblimp_ota.yaml
+include: _template_yaml
+dataset_name: ota
+task: multiblimp_ota
--- a/lm_eval/tasks/multiblimp/multiblimp_pcm.yaml
+++ b/lm_eval/tasks/multiblimp/multiblimp_pcm.yaml
+include: _template_yaml
+dataset_name: pcm
+task: multiblimp_pcm
--- a/lm_eval/tasks/multiblimp/multiblimp_pol.yaml
+++ b/lm_eval/tasks/multiblimp/multiblimp_pol.yaml
+include: _template_yaml
+dataset_name: pol
+task: multiblimp_pol
--- a/lm_eval/tasks/multiblimp/multiblimp_por.yaml
+++ b/lm_eval/tasks/multiblimp/multiblimp_por.yaml
+include: _template_yaml
+dataset_name: por
+task: multiblimp_por
--- a/lm_eval/tasks/multiblimp/multiblimp_quc.yaml
+++ b/lm_eval/tasks/multiblimp/multiblimp_quc.yaml
+include: _template_yaml
+dataset_name: quc
+task: multiblimp_quc
--- a/lm_eval/tasks/multiblimp/multiblimp_ron.yaml
+++ b/lm_eval/tasks/multiblimp/multiblimp_ron.yaml
+include: _template_yaml
+dataset_name: ron
+task: multiblimp_ron
--- a/lm_eval/tasks/multiblimp/multiblimp_rus.yaml
+++ b/lm_eval/tasks/multiblimp/multiblimp_rus.yaml
+include: _template_yaml
+dataset_name: rus
+task: multiblimp_rus
--- a/lm_eval/tasks/multiblimp/multiblimp_sah.yaml
+++ b/lm_eval/tasks/multiblimp/multiblimp_sah.yaml
+include: _template_yaml
+dataset_name: sah
+task: multiblimp_sah
--- a/lm_eval/tasks/multiblimp/multiblimp_san.yaml
+++ b/lm_eval/tasks/multiblimp/multiblimp_san.yaml
+include: _template_yaml
+dataset_name: san
+task: multiblimp_san