Merge branch 'upstream' into 'mmlu-pro'

add tokenizer logs info (#1731)

See merge request shijie.yu/lm-evaluation-harness!4

lm_eval/tasks/arabicmmlu/arabicmmlu_middle_natural_science.yaml

 "dataset_name": "Middle Natural Science"
-"group": "arabicmmlu_stem"
-"group_alias": "stem"
-"include": "_default_template_yaml"
+"tag": "arabicmmlu_stem_tasks"
+"include": "_default_arabicmmlu_template_yaml"
 "task": "arabicmmlu_middle_natural_science"
 "task_alias": "Middle Natural Science"

lm_eval/tasks/arabicmmlu/arabicmmlu_middle_social_science.yaml

 "dataset_name": "Middle Social Science"
-"group": "arabicmmlu_social_science"
-"group_alias": "social science"
-"include": "_default_template_yaml"
+"tag": "arabicmmlu_social_science_tasks"
+"include": "_default_arabicmmlu_template_yaml"
 "task": "arabicmmlu_middle_social_science"
 "task_alias": "Middle Social Science"

lm_eval/tasks/arabicmmlu/arabicmmlu_primary_arabic_language.yaml

 "dataset_name": "Primary Arabic Language"
-"group": "arabicmmlu_language"
-"group_alias": "language"
-"include": "_default_template_yaml"
+"tag": "arabicmmlu_language_tasks"
+"include": "_default_arabicmmlu_template_yaml"
 "task": "arabicmmlu_primary_arabic_language"
 "task_alias": "Primary Arabic Language"

lm_eval/tasks/arabicmmlu/arabicmmlu_primary_computer_science.yaml

 "dataset_name": "Primary Computer Science"
-"group": "arabicmmlu_stem"
-"group_alias": "stem"
-"include": "_default_template_yaml"
+"tag": "arabicmmlu_stem_tasks"
+"include": "_default_arabicmmlu_template_yaml"
 "task": "arabicmmlu_primary_computer_science"
 "task_alias": "Primary Computer Science"

lm_eval/tasks/arabicmmlu/arabicmmlu_primary_general_knowledge.yaml

 "dataset_name": "Primary General Knowledge"
-"group": "arabicmmlu_other"
-"group_alias": "other"
-"include": "_default_template_yaml"
+"tag": "arabicmmlu_other_tasks"
+"include": "_default_arabicmmlu_template_yaml"
 "task": "arabicmmlu_primary_general_knowledge"
 "task_alias": "Primary General Knowledge"

lm_eval/tasks/arabicmmlu/arabicmmlu_primary_geography.yaml

 "dataset_name": "Primary Geography"
-"group": "arabicmmlu_social_science"
-"group_alias": "social science"
-"include": "_default_template_yaml"
+"tag": "arabicmmlu_social_science_tasks"
+"include": "_default_arabicmmlu_template_yaml"
 "task": "arabicmmlu_primary_geography"
 "task_alias": "Primary Geography"

lm_eval/tasks/arabicmmlu/arabicmmlu_primary_history.yaml

 "dataset_name": "Primary History"
-"group": "arabicmmlu_humanities"
-"group_alias": "humanities"
-"include": "_default_template_yaml"
+"tag": "arabicmmlu_humanities_tasks"
+"include": "_default_arabicmmlu_template_yaml"
 "task": "arabicmmlu_primary_history"
 "task_alias": "Primary History"

lm_eval/tasks/arabicmmlu/arabicmmlu_primary_islamic_studies.yaml

 "dataset_name": "Primary Islamic Studies"
-"group": "arabicmmlu_humanities"
-"group_alias": "humanities"
-"include": "_default_template_yaml"
+"tag": "arabicmmlu_humanities_tasks"
+"include": "_default_arabicmmlu_template_yaml"
 "task": "arabicmmlu_primary_islamic_studies"
 "task_alias": "Primary Islamic Studies"

lm_eval/tasks/arabicmmlu/arabicmmlu_primary_math.yaml

 "dataset_name": "Primary Math"
-"group": "arabicmmlu_stem"
-"group_alias": "stem"
-"include": "_default_template_yaml"
+"tag": "arabicmmlu_stem_tasks"
+"include": "_default_arabicmmlu_template_yaml"
 "task": "arabicmmlu_primary_math"
 "task_alias": "Primary Math"

lm_eval/tasks/arabicmmlu/arabicmmlu_primary_natural_science.yaml

 "dataset_name": "Primary Natural Science"
-"group": "arabicmmlu_stem"
-"group_alias": "stem"
-"include": "_default_template_yaml"
+"tag": "arabicmmlu_stem_tasks"
+"include": "_default_arabicmmlu_template_yaml"
 "task": "arabicmmlu_primary_natural_science"
 "task_alias": "Primary Natural Science"

lm_eval/tasks/arabicmmlu/arabicmmlu_primary_social_science.yaml

 "dataset_name": "Primary Social Science"
-"group": "arabicmmlu_social_science"
-"group_alias": "social science"
-"include": "_default_template_yaml"
+"tag": "arabicmmlu_social_science_tasks"
+"include": "_default_arabicmmlu_template_yaml"
 "task": "arabicmmlu_primary_social_science"
 "task_alias": "Primary Social Science"

lm_eval/tasks/arabicmmlu/arabicmmlu_prof_law.yaml

 "dataset_name": "Prof Law"
-"group": "arabicmmlu_humanities"
-"group_alias": "humanities"
-"include": "_default_template_yaml"
+"tag": "arabicmmlu_humanities_tasks"
+"include": "_default_arabicmmlu_template_yaml"
 "task": "arabicmmlu_prof_law"
 "task_alias": "Prof Law"

lm_eval/tasks/arabicmmlu/arabicmmlu_univ_accounting.yaml

 "dataset_name": "Univ Accounting"
-"group": "arabicmmlu_social_science"
-"group_alias": "social science"
-"include": "_default_template_yaml"
+"tag": "arabicmmlu_social_science_tasks"
+"include": "_default_arabicmmlu_template_yaml"
 "task": "arabicmmlu_univ_accounting"
 "task_alias": "Univ Accounting"

lm_eval/tasks/arabicmmlu/arabicmmlu_univ_computer_science.yaml

 "dataset_name": "Univ Computer Science"
-"group": "arabicmmlu_stem"
-"group_alias": "stem"
-"include": "_default_template_yaml"
+"tag": "arabicmmlu_stem_tasks"
+"include": "_default_arabicmmlu_template_yaml"
 "task": "arabicmmlu_univ_computer_science"
 "task_alias": "Univ Computer Science"

lm_eval/tasks/arabicmmlu/arabicmmlu_univ_economics.yaml

 "dataset_name": "Univ Economics"
-"group": "arabicmmlu_social_science"
-"group_alias": "social science"
-"include": "_default_template_yaml"
+"tag": "arabicmmlu_social_science_tasks"
+"include": "_default_arabicmmlu_template_yaml"
 "task": "arabicmmlu_univ_economics"
 "task_alias": "Univ Economics"

lm_eval/tasks/arabicmmlu/arabicmmlu_univ_management.yaml

 "dataset_name": "Univ Management"
-"group": "arabicmmlu_other"
-"group_alias": "other"
-"include": "_default_template_yaml"
+"tag": "arabicmmlu_other_tasks"
+"include": "_default_arabicmmlu_template_yaml"
 "task": "arabicmmlu_univ_management"
 "task_alias": "Univ Management"

lm_eval/tasks/arabicmmlu/arabicmmlu_univ_political_science.yaml

 "dataset_name": "Univ Political Science"
-"group": "arabicmmlu_social_science"
-"group_alias": "social science"
-"include": "_default_template_yaml"
+"tag": "arabicmmlu_social_science_tasks"
+"include": "_default_arabicmmlu_template_yaml"
 "task": "arabicmmlu_univ_political_science"
 "task_alias": "Univ Political Science"

lm_eval/tasks/arabicmmlu/utils.py

-PROMPT = 'This is a {}. Select the correct answer!\n\nQuestion: {}\n{}\n\nAnswer:'
+PROMPT = "This is a {}. Select the correct answer!\n\nQuestion: {}\n{}\n\nAnswer:"
 
 level_en = {
-        'Primary': 'primary school',
-        'Middle': 'middle school',
-        'High': 'high school',
-        'Univ': 'university',
-        'Prof': 'professional',
+    "Primary": "primary school",
+    "Middle": "middle school",
+    "High": "high school",
+    "Univ": "university",
+    "Prof": "professional",
 }
 
-alpa = ['A.', 'B.', 'C.', 'D.', 'E.']
+alpa = ["A.", "B.", "C.", "D.", "E."]
 
 
 def doc_to_text(doc):
@@ -17,22 +17,28 @@ def doc_to_text(doc):
     https://github.com/mbzuai-nlp/ArabicMMLU/blob/main/util_prompt.py
     """
 
-    level = "" if not doc['Level'] else " for " + level_en[doc['Level']]
-    country = "" if not doc['Country'] else " in " + doc['Country']
+    level = "" if not doc["Level"] else " for " + level_en[doc["Level"]]
+    country = "" if not doc["Country"] else " in " + doc["Country"]
     main_meta_data = f"{doc['Subject']} question{level}{country}"
 
-    question = doc['Question'] if doc['Context']=="" else f"{doc['Context']}\n\n{doc['Question']}"
+    question = (
+        doc["Question"]
+        if doc["Context"] == ""
+        else f"{doc['Context']}\n\n{doc['Question']}"
+    )
 
     options = []
-    for i, opt in enumerate(['Option 1', 'Option 2', 'Option 3', 'Option 4', 'Option 5']):
+    for i, opt in enumerate(
+        ["Option 1", "Option 2", "Option 3", "Option 4", "Option 5"]
+    ):
         if not doc[opt]:
             break
         options.append(f"{alpa[i]} {doc[opt]}")
 
-    doc_text = PROMPT.format(main_meta_data, question, '\n'.join(options))
-    
+    doc_text = PROMPT.format(main_meta_data, question, "\n".join(options))
+
     return doc_text
 
 
 def doc_to_choice(doc):
-    return [alpa[i][0] for i in range(5) if doc[f'Option {i+1}']]
\ No newline at end of file
+    return [alpa[i][0] for i in range(5) if doc[f"Option {i+1}"]]

lm_eval/tasks/arc/README.md

@@ -29,10 +29,14 @@ Homepage: https://allenai.org/data/arc
 }
 ```
 
-### Groups and Tasks
+### Groups, Tags, and Tasks
 
 #### Groups
 
+None.
+
+#### Tags
+
 * `ai2_arc`: Evaluates `arc_easy` and `arc_challenge`
 
 #### Tasks

lm_eval/tasks/arc/arc_easy.yaml

-group:
+tag:
   - ai2_arc
 task: arc_easy
 dataset_path: allenai/ai2_arc