Merge branch 'main' of https://github.com/EleutherAI/lm-evaluation-harness into mela

lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10cm_medium.yaml

+dataset_name: icd10cm_medium
+include: _default_template_yaml
+tag: med_concepts_qa_icd10cm_tasks
+task: med_concepts_qa_icd10cm_medium
+task_alias: icd10cm_medium

lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10proc_easy.yaml

+dataset_name: icd10proc_easy
+include: _default_template_yaml
+tag: med_concepts_qa_icd10proc_tasks
+task: med_concepts_qa_icd10proc_easy
+task_alias: icd10proc_easy

lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10proc_hard.yaml

+dataset_name: icd10proc_hard
+include: _default_template_yaml
+tag: med_concepts_qa_icd10proc_tasks
+task: med_concepts_qa_icd10proc_hard
+task_alias: icd10proc_hard

lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10proc_medium.yaml

+dataset_name: icd10proc_medium
+include: _default_template_yaml
+tag: med_concepts_qa_icd10proc_tasks
+task: med_concepts_qa_icd10proc_medium
+task_alias: icd10proc_medium

lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_easy.yaml

+dataset_name: icd9cm_easy
+include: _default_template_yaml
+tag: med_concepts_qa_icd9cm_tasks
+task: med_concepts_qa_icd9cm_easy
+task_alias: icd9cm_easy

lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_hard.yaml

+dataset_name: icd9cm_hard
+include: _default_template_yaml
+tag: med_concepts_qa_icd9cm_tasks
+task: med_concepts_qa_icd9cm_hard
+task_alias: icd9cm_hard

lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_medium.yaml

+dataset_name: icd9cm_medium
+include: _default_template_yaml
+tag: med_concepts_qa_icd9cm_tasks
+task: med_concepts_qa_icd9cm_medium
+task_alias: icd9cm_medium

lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9proc_easy.yaml

+dataset_name: icd9proc_easy
+include: _default_template_yaml
+tag: med_concepts_qa_icd9proc_tasks
+task: med_concepts_qa_icd9proc_easy
+task_alias: icd9proc_easy

lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9proc_hard.yaml

+dataset_name: icd9proc_hard
+include: _default_template_yaml
+tag: med_concepts_qa_icd9proc_tasks
+task: med_concepts_qa_icd9proc_hard
+task_alias: icd9proc_hard

lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9proc_medium.yaml

+dataset_name: icd9proc_medium
+include: _default_template_yaml
+tag: med_concepts_qa_icd9proc_tasks
+task: med_concepts_qa_icd9proc_medium
+task_alias: icd9proc_medium

lm_eval/tasks/mmlu/_generate_configs.py

@@ -110,10 +110,9 @@ if __name__ == "__main__":
 
         yaml_dict = {
             "include": base_yaml_name,
-            "group": f"mmlu_{args.task_prefix}_{category}"
+            "tag": f"mmlu_{args.task_prefix}_{category}"
             if args.task_prefix != ""
             else f"mmlu_{category}",
-            "group_alias": category.replace("_", " "),
             "task": f"mmlu_{args.task_prefix}_{subject}"
             if args.task_prefix != ""
             else f"mmlu_{subject}",

lm_eval/tasks/mmlu/continuation/_continuation_template_yaml

@@ -8,4 +8,6 @@ doc_to_text: "Question: {{question.strip()}}\nAnswer:"
 doc_to_choice: "{{choices}}"
 doc_to_target: "{{answer}}"
 metadata:
-  version: 0.0
+  version: 1.0
+dataset_kwargs:
+  trust_remote_code: true

lm_eval/tasks/mmlu/continuation/_mmlu.yaml

 group: mmlu_continuation
+group_alias: mmlu (continuation)
 task:
-  - mmlu_continuation_stem
-  - mmlu_continuation_other
-  - mmlu_continuation_social_sciences
-  - mmlu_continuation_humanities
+  - group: stem
+    task:
+      - mmlu_continuation_stem
+    aggregate_metric_list:
+      - metric: acc
+        weight_by_size: True
+  - group: other
+    task:
+      - mmlu_continuation_other
+    aggregate_metric_list:
+      - metric: acc
+        weight_by_size: True
+  - group: social sciences
+    task:
+      - mmlu_continuation_social_sciences
+    aggregate_metric_list:
+      - metric: acc
+        weight_by_size: True
+  - group: humanities
+    task:
+      - mmlu_continuation_humanities
+    aggregate_metric_list:
+      - metric: acc
+        weight_by_size: True
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 2

lm_eval/tasks/mmlu/continuation/mmlu_abstract_algebra.yaml

 "dataset_name": "abstract_algebra"
 "description": "The following are questions (with answers) about abstract\
   \ algebra.\n\n"
-"group": "mmlu_continuation_stem"
+"tag": "mmlu_continuation_stem"
 "include": "_continuation_template_yaml"
 "task": "mmlu_continuation_abstract_algebra"

lm_eval/tasks/mmlu/continuation/mmlu_anatomy.yaml

 "dataset_name": "anatomy"
 "description": "The following are questions (with answers) about anatomy.\n\
   \n"
-"group": "mmlu_continuation_stem"
+"tag": "mmlu_continuation_stem"
 "include": "_continuation_template_yaml"
 "task": "mmlu_continuation_anatomy"

lm_eval/tasks/mmlu/continuation/mmlu_astronomy.yaml

 "dataset_name": "astronomy"
 "description": "The following are questions (with answers) about astronomy.\n\
   \n"
-"group": "mmlu_continuation_stem"
+"tag": "mmlu_continuation_stem"
 "include": "_continuation_template_yaml"
 "task": "mmlu_continuation_astronomy"

lm_eval/tasks/mmlu/continuation/mmlu_business_ethics.yaml

 "dataset_name": "business_ethics"
 "description": "The following are questions (with answers) about business\
   \ ethics.\n\n"
-"group": "mmlu_continuation_other"
+"tag": "mmlu_continuation_other"
 "include": "_continuation_template_yaml"
 "task": "mmlu_continuation_business_ethics"

lm_eval/tasks/mmlu/continuation/mmlu_clinical_knowledge.yaml

 "dataset_name": "clinical_knowledge"
 "description": "The following are questions (with answers) about clinical\
   \ knowledge.\n\n"
-"group": "mmlu_continuation_other"
+"tag": "mmlu_continuation_other"
 "include": "_continuation_template_yaml"
 "task": "mmlu_continuation_clinical_knowledge"

lm_eval/tasks/mmlu/continuation/mmlu_college_biology.yaml

 "dataset_name": "college_biology"
 "description": "The following are questions (with answers) about college\
   \ biology.\n\n"
-"group": "mmlu_continuation_stem"
+"tag": "mmlu_continuation_stem"
 "include": "_continuation_template_yaml"
 "task": "mmlu_continuation_college_biology"

lm_eval/tasks/mmlu/continuation/mmlu_college_chemistry.yaml

 "dataset_name": "college_chemistry"
 "description": "The following are questions (with answers) about college\
   \ chemistry.\n\n"
-"group": "mmlu_continuation_stem"
+"tag": "mmlu_continuation_stem"
 "include": "_continuation_template_yaml"
 "task": "mmlu_continuation_college_chemistry"