solved merge conflict

lm_eval/tasks/bbh/cot_zeroshot/_cot_zeroshot_template_yaml

@@ -24,4 +24,4 @@ filter_list:
       - function: "take_first"
 num_fewshot: 0
 metadata:
-  - version: 0
+  version: 1.0

lm_eval/tasks/bbh/fewshot/_fewshot_template_yaml

@@ -18,4 +18,4 @@ generation_kwargs:
   temperature: 0.0
 num_fewshot: 0
 metadata:
-  - version: 0
+  version: 1.0

lm_eval/tasks/bbh/zeroshot/_zeroshot_template_yaml

@@ -18,4 +18,4 @@ generation_kwargs:
   temperature: 0.0
 num_fewshot: 0
 metadata:
-  - version: 0
+  version: 1.0

lm_eval/tasks/belebele/_default_template_yaml

 group: belebele
 dataset_path: facebook/belebele
-test_split: test
-fewshot_split: test
 fewshot_config:
   sampler: first_n
 output_type: multiple_choice
@@ -18,4 +16,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/belebele/_generate_configs.py

@@ -8,7 +8,7 @@ import requests
 
 from tqdm import tqdm
 
-from lm_eval.logger import eval_logger
+from lm_eval.utils import logging
 
 API_URL = "https://datasets-server.huggingface.co/splits?dataset=facebook/belebele"
 
@@ -23,7 +23,6 @@ def parse_args():
 
 
 if __name__ == "__main__":
-
     args = parse_args()
 
     # get filename of base_yaml so we can `"include": ` it in our other YAMLs.
@@ -40,8 +39,8 @@ if __name__ == "__main__":
     def query():
         response = requests.get(API_URL)
         return response.json()["splits"]
-
-    languages = [split["config"] for split in query()]
+    print(query())
+    languages = [split["split"] for split in query()]
 
     for lang in tqdm(languages):
         yaml_dict = {
@@ -49,11 +48,12 @@ if __name__ == "__main__":
             "task": f"belebele_{args.task_prefix}_{lang}"
             if args.task_prefix != ""
             else f"belebele_{lang}",
-            "dataset_name": lang,
+            "test_split": lang,
+            "fewshot_split":lang,
         }
 
         file_save_path = args.save_prefix_path + f"_{lang}.yaml"
-        eval_logger.info(f"Saving yaml for subset {lang} to {file_save_path}")
+        logging.info(f"Saving yaml for subset {lang} to {file_save_path}")
         with open(file_save_path, "w") as yaml_file:
             yaml.dump(
                 yaml_dict,

lm_eval/tasks/belebele/belebele_acm_Arab.yaml

-"dataset_name": "acm_Arab"
+"fewshot_split": "acm_Arab"
 "include": "_default_template_yaml"
 "task": "belebele_acm_Arab"
+"test_split": "acm_Arab"

lm_eval/tasks/belebele/belebele_afr_Latn.yaml

-"dataset_name": "afr_Latn"
+"fewshot_split": "afr_Latn"
 "include": "_default_template_yaml"
 "task": "belebele_afr_Latn"
+"test_split": "afr_Latn"

lm_eval/tasks/belebele/belebele_als_Latn.yaml

-"dataset_name": "als_Latn"
+"fewshot_split": "als_Latn"
 "include": "_default_template_yaml"
 "task": "belebele_als_Latn"
+"test_split": "als_Latn"

lm_eval/tasks/belebele/belebele_amh_Ethi.yaml

-"dataset_name": "amh_Ethi"
+"fewshot_split": "amh_Ethi"
 "include": "_default_template_yaml"
 "task": "belebele_amh_Ethi"
+"test_split": "amh_Ethi"

lm_eval/tasks/belebele/belebele_apc_Arab.yaml

-"dataset_name": "apc_Arab"
+"fewshot_split": "apc_Arab"
 "include": "_default_template_yaml"
 "task": "belebele_apc_Arab"
+"test_split": "apc_Arab"

lm_eval/tasks/belebele/belebele_arb_Arab.yaml

-"dataset_name": "arb_Arab"
+"fewshot_split": "arb_Arab"
 "include": "_default_template_yaml"
 "task": "belebele_arb_Arab"
+"test_split": "arb_Arab"

lm_eval/tasks/belebele/belebele_arb_Latn.yaml

-"dataset_name": "arb_Latn"
+"fewshot_split": "arb_Latn"
 "include": "_default_template_yaml"
 "task": "belebele_arb_Latn"
+"test_split": "arb_Latn"

lm_eval/tasks/belebele/belebele_ars_Arab.yaml

-"dataset_name": "ars_Arab"
+"fewshot_split": "ars_Arab"
 "include": "_default_template_yaml"
 "task": "belebele_ars_Arab"
+"test_split": "ars_Arab"

lm_eval/tasks/belebele/belebele_ary_Arab.yaml

-"dataset_name": "ary_Arab"
+"fewshot_split": "ary_Arab"
 "include": "_default_template_yaml"
 "task": "belebele_ary_Arab"
+"test_split": "ary_Arab"

lm_eval/tasks/belebele/belebele_arz_Arab.yaml

-"dataset_name": "arz_Arab"
+"fewshot_split": "arz_Arab"
 "include": "_default_template_yaml"
 "task": "belebele_arz_Arab"
+"test_split": "arz_Arab"

lm_eval/tasks/belebele/belebele_asm_Beng.yaml

-"dataset_name": "asm_Beng"
+"fewshot_split": "asm_Beng"
 "include": "_default_template_yaml"
 "task": "belebele_asm_Beng"
+"test_split": "asm_Beng"

lm_eval/tasks/belebele/belebele_azj_Latn.yaml

-"dataset_name": "azj_Latn"
+"fewshot_split": "azj_Latn"
 "include": "_default_template_yaml"
 "task": "belebele_azj_Latn"
+"test_split": "azj_Latn"

lm_eval/tasks/belebele/belebele_bam_Latn.yaml

-"dataset_name": "bam_Latn"
+"fewshot_split": "bam_Latn"
 "include": "_default_template_yaml"
 "task": "belebele_bam_Latn"
+"test_split": "bam_Latn"

lm_eval/tasks/belebele/belebele_ben_Beng.yaml

-"dataset_name": "ben_Beng"
+"fewshot_split": "ben_Beng"
 "include": "_default_template_yaml"
 "task": "belebele_ben_Beng"
+"test_split": "ben_Beng"

lm_eval/tasks/belebele/belebele_ben_Latn.yaml

-"dataset_name": "ben_Latn"
+"fewshot_split": "ben_Latn"
 "include": "_default_template_yaml"
 "task": "belebele_ben_Latn"
+"test_split": "ben_Latn"