add group

lm_eval/tasks/mrl/mc/_generate_config.py

@@ -17,3 +17,15 @@ if __name__ == "__main__":
             f.write("include: '_template_mc'\n")
             f.write(f"task: mrl_{s}\n")
             f.write(f"dataset_name: {s}\n")
+
+with open(PARENT / "_global_piqa.yaml", "w") as f:
+    f.write("group: global_piqa\n")
+    f.write("task:\n")
+    for s in subsets:
+        f.write(f"  - mrl_{s}\n")
+    f.write("aggregate_metric_list:\n")
+    f.write("  - metric: acc\n")
+    f.write("    aggregation: mean\n")
+    f.write("    weight_by_size: true\n")
+    f.write("metadata:\n")
+    f.write("  version: 1.0\n")

lm_eval/tasks/mrl/mc/_global_piqa.yaml

+group: global_piqa
+task:
+  - mrl_acq_arab
+  - mrl_aeb_arab
+  - mrl_afb_arab
+  - mrl_als_latn
+  - mrl_amh_ethi
+  - mrl_apc_arab_jord
+  - mrl_apc_arab_leba
+  - mrl_apc_arab_pale
+  - mrl_apc_arab_syri
+  - mrl_arb_arab
+  - mrl_arq_arab
+  - mrl_ars_arab
+  - mrl_ary_arab
+  - mrl_arz_arab
+  - mrl_azj_latn
+  - mrl_bam_latn
+  - mrl_bel_cyrl
+  - mrl_ben_latn
+  - mrl_bho_deva
+  - mrl_bsk_arab
+  - mrl_bul_cyrl
+  - mrl_cat_latn
+  - mrl_ces_latn
+  - mrl_ckb_arab
+  - mrl_ckm_latn
+  - mrl_cmn_hans
+  - mrl_cmn_hant
+  - mrl_dhd_deva
+  - mrl_ell_grek
+  - mrl_eng_latn
+  - mrl_est_latn
+  - mrl_fao_latn
+  - mrl_fin_latn
+  - mrl_fra_latn_cana
+  - mrl_fra_latn_fran
+  - mrl_glg_latn
+  - mrl_guj_gujr
+  - mrl_hau_latn
+  - mrl_haw_latn
+  - mrl_heb_hebr
+  - mrl_hrv_latn
+  - mrl_hun_latn
+  - mrl_hye_armn
+  - mrl_ibo_latn
+  - mrl_idu_latn
+  - mrl_ind_latn
+  - mrl_isl_latn
+  - mrl_iso_latn
+  - mrl_ita_latn
+  - mrl_jav_latn
+  - mrl_jpn_jpan
+  - mrl_kat_geor
+  - mrl_kaz_cyrl
+  - mrl_kir_cyrl
+  - mrl_kor_hang
+  - mrl_lit_latn
+  - mrl_mar_deva
+  - mrl_mkd_cyrl
+  - mrl_mni_beng
+  - mrl_nag_latn
+  - mrl_nld_latn
+  - mrl_nno_latn
+  - mrl_nob_latn
+  - mrl_npi_deva
+  - mrl_pcm_latn
+  - mrl_pes_arab
+  - mrl_pol_latn
+  - mrl_por_latn_braz
+  - mrl_por_latn_port
+  - mrl_ron_latn
+  - mrl_rwr_deva
+  - mrl_sin_sinh
+  - mrl_slk_latn
+  - mrl_slk_latn_sari
+  - mrl_slv_latn
+  - mrl_slv_latn_cerk
+  - mrl_snd_arab
+  - mrl_snd_deva
+  - mrl_spa_latn_peru
+  - mrl_srp_cyrl
+  - mrl_srp_latn
+  - mrl_swe_latn
+  - mrl_tam_taml
+  - mrl_tgl_latn
+  - mrl_tha_thai
+  - mrl_tur_latn
+  - mrl_uig_arab
+  - mrl_ukr_cyrl
+  - mrl_urd_arab
+  - mrl_urd_latn
+  - mrl_urh_latn
+  - mrl_uzn_latn
+  - mrl_vie_latn
+  - mrl_yor_latn
+  - mrl_yue_hant
+  - mrl_zsm_latn
+  - mrl_zul_latn
+aggregate_metric_list:
+  - metric: acc
+    aggregation: mean
+    weight_by_size: true
+metadata:
+  version: 1.0