Merge branch 'group-agg-rework' of...

Merge branch 'group-agg-rework' of https://github.com/EleutherAI/lm-evaluation-harness into multiprompt

lm_eval/tasks/mmlu/default/mmlu_philosophy.yaml

 "dataset_name": "philosophy"
 "description": "The following are multiple choice questions (with answers) about philosophy.\n\
   \n"
-"group": "mmlu_humanities_tasks"
+"tag": "mmlu_humanities_tasks"
 "include": "_default_template_yaml"
 "task": "mmlu_philosophy"
 "task_alias": "philosophy"

lm_eval/tasks/mmlu/default/mmlu_prehistory.yaml

 "dataset_name": "prehistory"
 "description": "The following are multiple choice questions (with answers) about prehistory.\n\
   \n"
-"group": "mmlu_humanities_tasks"
+"tag": "mmlu_humanities_tasks"
 "include": "_default_template_yaml"
 "task": "mmlu_prehistory"
 "task_alias": "prehistory"

lm_eval/tasks/mmlu/default/mmlu_professional_accounting.yaml

 "dataset_name": "professional_accounting"
 "description": "The following are multiple choice questions (with answers) about professional\
   \ accounting.\n\n"
-"group": "mmlu_other_tasks"
+"tag": "mmlu_other_tasks"
 "include": "_default_template_yaml"
 "task": "mmlu_professional_accounting"
 "task_alias": "professional_accounting"

lm_eval/tasks/mmlu/default/mmlu_professional_law.yaml

 "dataset_name": "professional_law"
 "description": "The following are multiple choice questions (with answers) about professional\
   \ law.\n\n"
-"group": "mmlu_humanities_tasks"
+"tag": "mmlu_humanities_tasks"
 "include": "_default_template_yaml"
 "task": "mmlu_professional_law"
 "task_alias": "professional_law"

lm_eval/tasks/mmlu/default/mmlu_professional_medicine.yaml

 "dataset_name": "professional_medicine"
 "description": "The following are multiple choice questions (with answers) about professional\
   \ medicine.\n\n"
-"group": "mmlu_other_tasks"
+"tag": "mmlu_other_tasks"
 "include": "_default_template_yaml"
 "task": "mmlu_professional_medicine"
 "task_alias": "professional_medicine"

lm_eval/tasks/mmlu/default/mmlu_professional_psychology.yaml

 "dataset_name": "professional_psychology"
 "description": "The following are multiple choice questions (with answers) about professional\
   \ psychology.\n\n"
-"group": "mmlu_social_sciences_tasks"
+"tag": "mmlu_social_sciences_tasks"
 "include": "_default_template_yaml"
 "task": "mmlu_professional_psychology"
 "task_alias": "professional_psychology"

lm_eval/tasks/mmlu/default/mmlu_public_relations.yaml

 "dataset_name": "public_relations"
 "description": "The following are multiple choice questions (with answers) about public\
   \ relations.\n\n"
-"group": "mmlu_social_sciences_tasks"
+"tag": "mmlu_social_sciences_tasks"
 "include": "_default_template_yaml"
 "task": "mmlu_public_relations"
 "task_alias": "public_relations"

lm_eval/tasks/mmlu/default/mmlu_security_studies.yaml

 "dataset_name": "security_studies"
 "description": "The following are multiple choice questions (with answers) about security\
   \ studies.\n\n"
-"group": "mmlu_social_sciences_tasks"
+"tag": "mmlu_social_sciences_tasks"
 "include": "_default_template_yaml"
 "task": "mmlu_security_studies"
 "task_alias": "security_studies"

lm_eval/tasks/mmlu/default/mmlu_sociology.yaml

 "dataset_name": "sociology"
 "description": "The following are multiple choice questions (with answers) about sociology.\n\
   \n"
-"group": "mmlu_social_sciences_tasks"
+"tag": "mmlu_social_sciences_tasks"
 "include": "_default_template_yaml"
 "task": "mmlu_sociology"
 "task_alias": "sociology"

lm_eval/tasks/mmlu/default/mmlu_us_foreign_policy.yaml

 "dataset_name": "us_foreign_policy"
 "description": "The following are multiple choice questions (with answers) about us\
   \ foreign policy.\n\n"
-"group": "mmlu_social_sciences_tasks"
+"tag": "mmlu_social_sciences_tasks"
 "include": "_default_template_yaml"
 "task": "mmlu_us_foreign_policy"
 "task_alias": "us_foreign_policy"

lm_eval/tasks/mmlu/default/mmlu_virology.yaml

 "dataset_name": "virology"
 "description": "The following are multiple choice questions (with answers) about virology.\n\
   \n"
-"group": "mmlu_other_tasks"
+"tag": "mmlu_other_tasks"
 "include": "_default_template_yaml"
 "task": "mmlu_virology"
 "task_alias": "virology"

lm_eval/tasks/mmlu/default/mmlu_world_religions.yaml

 "dataset_name": "world_religions"
 "description": "The following are multiple choice questions (with answers) about world\
   \ religions.\n\n"
-"group": "mmlu_humanities_tasks"
+"tag": "mmlu_humanities_tasks"
 "include": "_default_template_yaml"
 "task": "mmlu_world_religions"
 "task_alias": "world_religions"

lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu.yaml

 group: mmlu_flan_cot_fewshot
+group_alias: mmlu (flan style, fewshot cot)
 task:
-  - mmlu_flan_cot_fewshot_stem
-  - mmlu_flan_cot_fewshot_other
-  - mmlu_flan_cot_fewshot_social_sciences
-  - mmlu_flan_cot_fewshot_humanities
-group_config:
-  aggregate_metric: True
-  weight_by_size: True
+  - group: stem
+    task:
+      - mmlu_flan_cot_fewshot_stem
+    aggregate_metric_list:
+      - metric: acc
+        weight_by_size: True
+  - group: other
+    task:
+      - mmlu_flan_cot_fewshot_other
+    aggregate_metric_list:
+      - metric: acc
+        weight_by_size: True
+  - group: social sciences
+    task:
+      - mmlu_flan_cot_fewshot_social_sciences
+    aggregate_metric_list:
+      - metric: acc
+        weight_by_size: True
+  - group: humanities
+    task:
+      - mmlu_flan_cot_fewshot_humanities
+    aggregate_metric_list:
+      - metric: acc
+        weight_by_size: True
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 1

lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu_flan_cot_fewshot_template_yaml

@@ -27,3 +27,5 @@ metric_list:
     ignore_punctuation: true
 metadata:
   version: 1.0
+dataset_kwargs:
+  trust_remote_code: true

lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_abstract_algebra.yaml

@@ -54,6 +54,6 @@ fewshot_config:
       not have any roots. For c = 2 the polynomial x^2 + 2 has two roots at x = 1
       and x = 2. Hence Z_3[x]/(x^2 + c) is a field if and only if c = 1. The answer
       is (B).'
-group: mmlu_flan_cot_fewshot_stem
+tag: mmlu_flan_cot_fewshot_stem
 include: _mmlu_flan_cot_fewshot_template_yaml
 task: mmlu_flan_cot_fewshot_abstract_algebra

lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_anatomy.yaml

@@ -70,6 +70,6 @@ fewshot_config:
       \ origin of the hyoid bone are the second and the third pharyngeal arches\u2014\
       this information is covered in the last option (D). Therefore, we conclude that\
       \ (D) must be the correct answer. The answer is (D).\n\n"
-group: mmlu_flan_cot_fewshot_stem
+tag: mmlu_flan_cot_fewshot_stem
 include: _mmlu_flan_cot_fewshot_template_yaml
 task: mmlu_flan_cot_fewshot_anatomy

lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_astronomy.yaml

@@ -65,6 +65,6 @@ fewshot_config:
       because it explains that the surface is red due to the rusted materials on the
       surface and the red color comes from the rust. So the correct option is (A).
       The answer is (A).'
-group: mmlu_flan_cot_fewshot_stem
+tag: mmlu_flan_cot_fewshot_stem
 include: _mmlu_flan_cot_fewshot_template_yaml
 task: mmlu_flan_cot_fewshot_astronomy

lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_business_ethics.yaml

@@ -70,6 +70,6 @@ fewshot_config:
       \ moral arguments relating to: negative *externalities*, the *power* that corporations\
       \ possess and the *mutual independence* of business and society. The answer\
       \ is (D).\n\n"
-group: mmlu_flan_cot_fewshot_other
+tag: mmlu_flan_cot_fewshot_other
 include: _mmlu_flan_cot_fewshot_template_yaml
 task: mmlu_flan_cot_fewshot_business_ethics

lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_clinical_knowledge.yaml

@@ -43,6 +43,6 @@ fewshot_config:
     target: 'Let''s think step by step. We refer to Wikipedia articles on clinical
       knowledge for help. The energy for muscular contraction is provided by ATP (adenosine
       triphosphate), which is the powerhouse of the cell. The answer is (A).'
-group: mmlu_flan_cot_fewshot_other
+tag: mmlu_flan_cot_fewshot_other
 include: _mmlu_flan_cot_fewshot_template_yaml
 task: mmlu_flan_cot_fewshot_clinical_knowledge

lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_biology.yaml

@@ -70,6 +70,6 @@ fewshot_config:
       that have different origins, which is not the case for the human and bird forearms,
       which rules out (D). Humans and birds do belong to the same clade - a group
       of organisms composed of a common ancestor. The answer is (C).'
-group: mmlu_flan_cot_fewshot_stem
+tag: mmlu_flan_cot_fewshot_stem
 include: _mmlu_flan_cot_fewshot_template_yaml
 task: mmlu_flan_cot_fewshot_college_biology