fixed belebele (#1267)

lm_eval/tasks/belebele/_default_template_yaml

 group: belebele
 dataset_path: facebook/belebele
-test_split: test
-fewshot_split: test
 fewshot_config:
   sampler: first_n
 output_type: multiple_choice

lm_eval/tasks/belebele/_generate_configs.py

@@ -8,7 +8,7 @@ import requests
 
 from tqdm import tqdm
 
-from lm_eval.logger import eval_logger
+from lm_eval.utils import logging
 
 API_URL = "https://datasets-server.huggingface.co/splits?dataset=facebook/belebele"
 
@@ -39,8 +39,8 @@ if __name__ == "__main__":
     def query():
         response = requests.get(API_URL)
         return response.json()["splits"]
-
-    languages = [split["config"] for split in query()]
+    print(query())
+    languages = [split["split"] for split in query()]
 
     for lang in tqdm(languages):
         yaml_dict = {
@@ -48,11 +48,12 @@ if __name__ == "__main__":
             "task": f"belebele_{args.task_prefix}_{lang}"
             if args.task_prefix != ""
             else f"belebele_{lang}",
-            "dataset_name": lang,
+            "test_split": lang,
+            "fewshot_split":lang,
         }
 
         file_save_path = args.save_prefix_path + f"_{lang}.yaml"
-        eval_logger.info(f"Saving yaml for subset {lang} to {file_save_path}")
+        logging.info(f"Saving yaml for subset {lang} to {file_save_path}")
         with open(file_save_path, "w") as yaml_file:
             yaml.dump(
                 yaml_dict,

lm_eval/tasks/belebele/belebele_acm_Arab.yaml

-"dataset_name": "acm_Arab"
+"fewshot_split": "acm_Arab"
 "include": "_default_template_yaml"
 "task": "belebele_acm_Arab"
+"test_split": "acm_Arab"

lm_eval/tasks/belebele/belebele_afr_Latn.yaml

-"dataset_name": "afr_Latn"
+"fewshot_split": "afr_Latn"
 "include": "_default_template_yaml"
 "task": "belebele_afr_Latn"
+"test_split": "afr_Latn"

lm_eval/tasks/belebele/belebele_als_Latn.yaml

-"dataset_name": "als_Latn"
+"fewshot_split": "als_Latn"
 "include": "_default_template_yaml"
 "task": "belebele_als_Latn"
+"test_split": "als_Latn"

lm_eval/tasks/belebele/belebele_amh_Ethi.yaml

-"dataset_name": "amh_Ethi"
+"fewshot_split": "amh_Ethi"
 "include": "_default_template_yaml"
 "task": "belebele_amh_Ethi"
+"test_split": "amh_Ethi"

lm_eval/tasks/belebele/belebele_apc_Arab.yaml

-"dataset_name": "apc_Arab"
+"fewshot_split": "apc_Arab"
 "include": "_default_template_yaml"
 "task": "belebele_apc_Arab"
+"test_split": "apc_Arab"

lm_eval/tasks/belebele/belebele_arb_Arab.yaml

-"dataset_name": "arb_Arab"
+"fewshot_split": "arb_Arab"
 "include": "_default_template_yaml"
 "task": "belebele_arb_Arab"
+"test_split": "arb_Arab"

lm_eval/tasks/belebele/belebele_arb_Latn.yaml

-"dataset_name": "arb_Latn"
+"fewshot_split": "arb_Latn"
 "include": "_default_template_yaml"
 "task": "belebele_arb_Latn"
+"test_split": "arb_Latn"

lm_eval/tasks/belebele/belebele_ars_Arab.yaml

-"dataset_name": "ars_Arab"
+"fewshot_split": "ars_Arab"
 "include": "_default_template_yaml"
 "task": "belebele_ars_Arab"
+"test_split": "ars_Arab"

lm_eval/tasks/belebele/belebele_ary_Arab.yaml

-"dataset_name": "ary_Arab"
+"fewshot_split": "ary_Arab"
 "include": "_default_template_yaml"
 "task": "belebele_ary_Arab"
+"test_split": "ary_Arab"

lm_eval/tasks/belebele/belebele_arz_Arab.yaml

-"dataset_name": "arz_Arab"
+"fewshot_split": "arz_Arab"
 "include": "_default_template_yaml"
 "task": "belebele_arz_Arab"
+"test_split": "arz_Arab"

lm_eval/tasks/belebele/belebele_asm_Beng.yaml

-"dataset_name": "asm_Beng"
+"fewshot_split": "asm_Beng"
 "include": "_default_template_yaml"
 "task": "belebele_asm_Beng"
+"test_split": "asm_Beng"

lm_eval/tasks/belebele/belebele_azj_Latn.yaml

-"dataset_name": "azj_Latn"
+"fewshot_split": "azj_Latn"
 "include": "_default_template_yaml"
 "task": "belebele_azj_Latn"
+"test_split": "azj_Latn"

lm_eval/tasks/belebele/belebele_bam_Latn.yaml

-"dataset_name": "bam_Latn"
+"fewshot_split": "bam_Latn"
 "include": "_default_template_yaml"
 "task": "belebele_bam_Latn"
+"test_split": "bam_Latn"

lm_eval/tasks/belebele/belebele_ben_Beng.yaml

-"dataset_name": "ben_Beng"
+"fewshot_split": "ben_Beng"
 "include": "_default_template_yaml"
 "task": "belebele_ben_Beng"
+"test_split": "ben_Beng"

lm_eval/tasks/belebele/belebele_ben_Latn.yaml

-"dataset_name": "ben_Latn"
+"fewshot_split": "ben_Latn"
 "include": "_default_template_yaml"
 "task": "belebele_ben_Latn"
+"test_split": "ben_Latn"

lm_eval/tasks/belebele/belebele_bod_Tibt.yaml

-"dataset_name": "bod_Tibt"
+"fewshot_split": "bod_Tibt"
 "include": "_default_template_yaml"
 "task": "belebele_bod_Tibt"
+"test_split": "bod_Tibt"

lm_eval/tasks/belebele/belebele_bul_Cyrl.yaml

-"dataset_name": "bul_Cyrl"
+"fewshot_split": "bul_Cyrl"
 "include": "_default_template_yaml"
 "task": "belebele_bul_Cyrl"
+"test_split": "bul_Cyrl"

lm_eval/tasks/belebele/belebele_cat_Latn.yaml

-"dataset_name": "cat_Latn"
+"fewshot_split": "cat_Latn"
 "include": "_default_template_yaml"
 "task": "belebele_cat_Latn"
+"test_split": "cat_Latn"