merged main

lm_eval/tasks/okapi/hellaswag_multilingual/utils.py

-import datasets
 import re
 
+import datasets
+
 
 def preprocess(text):
     text = text.strip()

lm_eval/tasks/okapi/mmlu_multilingual/_generate_configs.py

-import yaml
 import datasets
-
+import yaml
 from tqdm import tqdm
 
 
 def main() -> None:
     dataset_path = "alexandrainst/m_mmlu"
 
-    # Removed hy and sk subdataset because the original dataset is broken
-    # I created this PR https://huggingface.co/datasets/alexandrainst/m_mmlu/discussions/3
-    # on the dataset for the authors, in case it will be accepeted the filter can be removed
-    keys_without_hy_sk = list(filter(lambda k: ('hy' not in k and  'sk' not in k),
-                                     datasets.get_dataset_infos(dataset_path).keys()))
-
-    for task in tqdm():
+    for task in tqdm(datasets.get_dataset_infos(dataset_path).keys()):
         file_name = f"m_mmlu_{task}.yaml"
         try:
             with open(f"{file_name}", "w") as f:
@@ -29,5 +22,6 @@ def main() -> None:
         except FileExistsError:
             pass
 
+
 if __name__ == "__main__":
     main()

lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_hy.yaml

+# Generated by _generate_configs.py
+dataset_name: hy
+include: _default_yaml
+task: m_mmlu_hy

lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_sk.yaml

+# Generated by _generate_configs.py
+dataset_name: sk
+include: _default_yaml
+task: m_mmlu_sk

lm_eval/tasks/okapi/truthfulqa_multilingual/_truthfulqa_mc1_yaml

@@ -4,7 +4,7 @@ dataset_path: null
 dataset_name: null
 output_type: multiple_choice
 training_split: null
-validation_split: validation
+validation_split: val
 test_split: null
 process_docs: !function utils.process_docs
 doc_to_text: "query"

lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ar_mc1.yaml

@@ -3,5 +3,5 @@ task: truthfulqa_ar_mc1
 dataset_path: alexandrainst/m_truthfulqa
 dataset_name: ar
 training_split: null
-validation_split: validation
+validation_split: val
 test_split: null

lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ar_mc2.yaml

@@ -3,5 +3,5 @@ task: truthfulqa_ar_mc2
 dataset_path: alexandrainst/m_truthfulqa
 dataset_name: ar
 training_split: null
-validation_split: validation
+validation_split: val
 test_split: null

lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_bn_mc1.yaml

@@ -3,5 +3,5 @@ task: truthfulqa_bn_mc1
 dataset_path: alexandrainst/m_truthfulqa
 dataset_name: bn
 training_split: null
-validation_split: validation
+validation_split: val
 test_split: null

lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_bn_mc2.yaml

@@ -3,5 +3,5 @@ task: truthfulqa_bn_mc2
 dataset_path: alexandrainst/m_truthfulqa
 dataset_name: bn
 training_split: null
-validation_split: validation
+validation_split: val
 test_split: null

lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ca_mc1.yaml

@@ -3,5 +3,5 @@ task: truthfulqa_ca_mc1
 dataset_path: alexandrainst/m_truthfulqa
 dataset_name: ca
 training_split: null
-validation_split: validation
+validation_split: val
 test_split: null

lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ca_mc2.yaml

@@ -3,5 +3,5 @@ task: truthfulqa_ca_mc2
 dataset_path: alexandrainst/m_truthfulqa
 dataset_name: ca
 training_split: null
-validation_split: validation
+validation_split: val
 test_split: null

lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_da_mc1.yaml

@@ -3,5 +3,5 @@ task: truthfulqa_da_mc1
 dataset_path: alexandrainst/m_truthfulqa
 dataset_name: da
 training_split: null
-validation_split: validation
+validation_split: val
 test_split: null

lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_da_mc2.yaml

@@ -3,5 +3,5 @@ task: truthfulqa_da_mc2
 dataset_path: alexandrainst/m_truthfulqa
 dataset_name: da
 training_split: null
-validation_split: validation
+validation_split: val
 test_split: null

lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_de_mc1.yaml

@@ -3,5 +3,5 @@ task: truthfulqa_de_mc1
 dataset_path: alexandrainst/m_truthfulqa
 dataset_name: de
 training_split: null
-validation_split: validation
+validation_split: val
 test_split: null

lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_de_mc2.yaml

@@ -3,5 +3,5 @@ task: truthfulqa_de_mc2
 dataset_path: alexandrainst/m_truthfulqa
 dataset_name: de
 training_split: null
-validation_split: validation
+validation_split: val
 test_split: null

lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_es_mc1.yaml

@@ -3,5 +3,5 @@ task: truthfulqa_es_mc1
 dataset_path: alexandrainst/m_truthfulqa
 dataset_name: es
 training_split: null
-validation_split: validation
+validation_split: val
 test_split: null

lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_es_mc2.yaml

@@ -3,5 +3,5 @@ task: truthfulqa_es_mc2
 dataset_path: alexandrainst/m_truthfulqa
 dataset_name: es
 training_split: null
-validation_split: validation
+validation_split: val
 test_split: null

lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_eu_mc1.yaml

@@ -3,5 +3,5 @@ task: truthfulqa_eu_mc1
 dataset_path: alexandrainst/m_truthfulqa
 dataset_name: eu
 training_split: null
-validation_split: validation
+validation_split: val
 test_split: null

lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_eu_mc2.yaml

@@ -3,5 +3,5 @@ task: truthfulqa_eu_mc2
 dataset_path: alexandrainst/m_truthfulqa
 dataset_name: eu
 training_split: null
-validation_split: validation
+validation_split: val
 test_split: null

lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_fr_mc1.yaml

@@ -3,5 +3,5 @@ task: truthfulqa_fr_mc1
 dataset_path: alexandrainst/m_truthfulqa
 dataset_name: fr
 training_split: null
-validation_split: validation
+validation_split: val
 test_split: null