Merge branch 'main' into multimodal-prototyping

lm_eval/tasks/leaderboard/musr/musr_object_placements.yaml

+include: "_template_yaml"
+task: leaderboard_musr_object_placements
+test_split: object_placements

lm_eval/tasks/leaderboard/musr/musr_team_allocation.yaml

+include: "_template_yaml"
+task: leaderboard_musr_team_allocation
+test_split: team_allocation

lm_eval/tasks/leaderboard/musr/utils.py

+import ast
+
+
+def doc_to_choice(doc):
+    """
+    Convert a doc to a choice.
+    """
+    return ast.literal_eval(doc["choices"])
+
+
+DOC_TO_TEXT = "{narrative}\n\n" "{question}\n\n" "{choices}\n" "Answer:"
+
+
+def doc_to_text(doc):
+    """
+    Convert a doc to text.
+    """
+    choices = ""
+    for i, choice in enumerate(ast.literal_eval(doc["choices"])):
+        choices += f"{i+1} - {choice}\n"
+
+    text = DOC_TO_TEXT.format(
+        narrative=doc["narrative"], question=doc["question"], choices=choices
+    )
+
+    return text

lm_eval/tasks/mathqa/mathqa.yaml

-group:
+tag:
   - math_word_problems
 task: mathqa
 dataset_path: math_qa

lm_eval/tasks/mmlu/_generate_configs.py

@@ -110,10 +110,9 @@ if __name__ == "__main__":
 
         yaml_dict = {
             "include": base_yaml_name,
-            "group": f"mmlu_{args.task_prefix}_{category}"
+            "tag": f"mmlu_{args.task_prefix}_{category}"
             if args.task_prefix != ""
             else f"mmlu_{category}",
-            "group_alias": category.replace("_", " "),
             "task": f"mmlu_{args.task_prefix}_{subject}"
             if args.task_prefix != ""
             else f"mmlu_{subject}",

lm_eval/tasks/mmlu/continuation/_mmlu.yaml

 group: mmlu_continuation
+group_alias: mmlu (continuation)
 task:
-  - mmlu_continuation_stem
-  - mmlu_continuation_other
-  - mmlu_continuation_social_sciences
-  - mmlu_continuation_humanities
+  - group: stem
+    task:
+      - mmlu_continuation_stem
+    aggregate_metric_list:
+      - metric: acc
+        weight_by_size: True
+  - group: other
+    task:
+      - mmlu_continuation_other
+    aggregate_metric_list:
+      - metric: acc
+        weight_by_size: True
+  - group: social sciences
+    task:
+      - mmlu_continuation_social_sciences
+    aggregate_metric_list:
+      - metric: acc
+        weight_by_size: True
+  - group: humanities
+    task:
+      - mmlu_continuation_humanities
+    aggregate_metric_list:
+      - metric: acc
+        weight_by_size: True
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 1

lm_eval/tasks/mmlu/continuation/mmlu_abstract_algebra.yaml

 "dataset_name": "abstract_algebra"
 "description": "The following are questions (with answers) about abstract\
   \ algebra.\n\n"
-"group": "mmlu_continuation_stem"
+"tag": "mmlu_continuation_stem"
 "include": "_continuation_template_yaml"
 "task": "mmlu_continuation_abstract_algebra"

lm_eval/tasks/mmlu/continuation/mmlu_anatomy.yaml

 "dataset_name": "anatomy"
 "description": "The following are questions (with answers) about anatomy.\n\
   \n"
-"group": "mmlu_continuation_stem"
+"tag": "mmlu_continuation_stem"
 "include": "_continuation_template_yaml"
 "task": "mmlu_continuation_anatomy"

lm_eval/tasks/mmlu/continuation/mmlu_astronomy.yaml

 "dataset_name": "astronomy"
 "description": "The following are questions (with answers) about astronomy.\n\
   \n"
-"group": "mmlu_continuation_stem"
+"tag": "mmlu_continuation_stem"
 "include": "_continuation_template_yaml"
 "task": "mmlu_continuation_astronomy"

lm_eval/tasks/mmlu/continuation/mmlu_business_ethics.yaml

 "dataset_name": "business_ethics"
 "description": "The following are questions (with answers) about business\
   \ ethics.\n\n"
-"group": "mmlu_continuation_other"
+"tag": "mmlu_continuation_other"
 "include": "_continuation_template_yaml"
 "task": "mmlu_continuation_business_ethics"

lm_eval/tasks/mmlu/continuation/mmlu_clinical_knowledge.yaml

 "dataset_name": "clinical_knowledge"
 "description": "The following are questions (with answers) about clinical\
   \ knowledge.\n\n"
-"group": "mmlu_continuation_other"
+"tag": "mmlu_continuation_other"
 "include": "_continuation_template_yaml"
 "task": "mmlu_continuation_clinical_knowledge"

lm_eval/tasks/mmlu/continuation/mmlu_college_biology.yaml

 "dataset_name": "college_biology"
 "description": "The following are questions (with answers) about college\
   \ biology.\n\n"
-"group": "mmlu_continuation_stem"
+"tag": "mmlu_continuation_stem"
 "include": "_continuation_template_yaml"
 "task": "mmlu_continuation_college_biology"

lm_eval/tasks/mmlu/continuation/mmlu_college_chemistry.yaml

 "dataset_name": "college_chemistry"
 "description": "The following are questions (with answers) about college\
   \ chemistry.\n\n"
-"group": "mmlu_continuation_stem"
+"tag": "mmlu_continuation_stem"
 "include": "_continuation_template_yaml"
 "task": "mmlu_continuation_college_chemistry"

lm_eval/tasks/mmlu/continuation/mmlu_college_computer_science.yaml

 "dataset_name": "college_computer_science"
 "description": "The following are questions (with answers) about college\
   \ computer science.\n\n"
-"group": "mmlu_continuation_stem"
+"tag": "mmlu_continuation_stem"
 "include": "_continuation_template_yaml"
 "task": "mmlu_continuation_college_computer_science"

lm_eval/tasks/mmlu/continuation/mmlu_college_mathematics.yaml

 "dataset_name": "college_mathematics"
 "description": "The following are questions (with answers) about college\
   \ mathematics.\n\n"
-"group": "mmlu_continuation_stem"
+"tag": "mmlu_continuation_stem"
 "include": "_continuation_template_yaml"
 "task": "mmlu_continuation_college_mathematics"

lm_eval/tasks/mmlu/continuation/mmlu_college_medicine.yaml

 "dataset_name": "college_medicine"
 "description": "The following are questions (with answers) about college\
   \ medicine.\n\n"
-"group": "mmlu_continuation_other"
+"tag": "mmlu_continuation_other"
 "include": "_continuation_template_yaml"
 "task": "mmlu_continuation_college_medicine"

lm_eval/tasks/mmlu/continuation/mmlu_college_physics.yaml

 "dataset_name": "college_physics"
 "description": "The following are questions (with answers) about college\
   \ physics.\n\n"
-"group": "mmlu_continuation_stem"
+"tag": "mmlu_continuation_stem"
 "include": "_continuation_template_yaml"
 "task": "mmlu_continuation_college_physics"

lm_eval/tasks/mmlu/continuation/mmlu_computer_security.yaml

 "dataset_name": "computer_security"
 "description": "The following are questions (with answers) about computer\
   \ security.\n\n"
-"group": "mmlu_continuation_stem"
+"tag": "mmlu_continuation_stem"
 "include": "_continuation_template_yaml"
 "task": "mmlu_continuation_computer_security"

lm_eval/tasks/mmlu/continuation/mmlu_conceptual_physics.yaml

 "dataset_name": "conceptual_physics"
 "description": "The following are questions (with answers) about conceptual\
   \ physics.\n\n"
-"group": "mmlu_continuation_stem"
+"tag": "mmlu_continuation_stem"
 "include": "_continuation_template_yaml"
 "task": "mmlu_continuation_conceptual_physics"

lm_eval/tasks/mmlu/continuation/mmlu_econometrics.yaml

 "dataset_name": "econometrics"
 "description": "The following are questions (with answers) about econometrics.\n\
   \n"
-"group": "mmlu_continuation_social_sciences"
+"tag": "mmlu_continuation_social_sciences"
 "include": "_continuation_template_yaml"
 "task": "mmlu_continuation_econometrics"