Merge branch 'smolrefact' into tasklist

# Conflicts:
#	lm_eval/__main__.py
#	lm_eval/api/group.py
#	lm_eval/api/task.py
#	lm_eval/evaluator_utils.py
#	lm_eval/tasks/__init__.py
#	lm_eval/utils.py
#	pyproject.toml

lm_eval/tasks/mmlu-redux/generative/mmlu_us_foreign_policy.yaml

+"dataset_name": "us_foreign_policy"
+"description": "The following are multiple choice questions (with answers) about us\
+  \ foreign policy.\n\n"
+"tag": "mmlu_social_sciences_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_us_foreign_policy_generative"
+"task_alias": "us_foreign_policy"

lm_eval/tasks/mmlu-redux/generative/mmlu_virology.yaml

+"dataset_name": "virology"
+"description": "The following are multiple choice questions (with answers) about virology.\n\
+  \n"
+"tag": "mmlu_other_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_virology_generative"
+"task_alias": "virology"

lm_eval/tasks/mmlu-redux/generative/mmlu_world_religions.yaml

+"dataset_name": "world_religions"
+"description": "The following are multiple choice questions (with answers) about world\
+  \ religions.\n\n"
+"tag": "mmlu_humanities_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_world_religions_generative"
+"task_alias": "world_religions"

lm_eval/tasks/mmlu/README.md

@@ -71,3 +71,6 @@ switch to original implementation
 
 ver 2: PR #2116
 add missing newline in description.
+
+PR #3137
+Fix `mmlu_continuation` subgroup names to fit other variants, and switch dataset from `hails/mmlu_no_train` to `cais/mmlu` in all subtasks.

lm_eval/tasks/mmlu/continuation/_continuation_template_yaml

-dataset_path: hails/mmlu_no_train # a copy of `cais/mmlu` with no auxiliary_train split
+dataset_path: cais/mmlu
 output_type: multiple_choice
 test_split: test
 fewshot_split: dev

lm_eval/tasks/mmlu/continuation/_mmlu.yaml

@@ -3,25 +3,25 @@ group_alias: mmlu (continuation)
 task:
   - group: stem
     task:
-      - mmlu_continuation_stem
+      - mmlu_stem_continuation
     aggregate_metric_list:
       - metric: acc
         weight_by_size: True
   - group: other
     task:
-      - mmlu_continuation_other
+      - mmlu_other_continuation
     aggregate_metric_list:
       - metric: acc
         weight_by_size: True
   - group: social sciences
     task:
-      - mmlu_continuation_social_sciences
+      - mmlu_social_sciences_continuation
     aggregate_metric_list:
       - metric: acc
         weight_by_size: True
   - group: humanities
     task:
-      - mmlu_continuation_humanities
+      - mmlu_humanities_continuation
     aggregate_metric_list:
       - metric: acc
         weight_by_size: True

lm_eval/tasks/mmlu/continuation/mmlu_abstract_algebra.yaml

 "dataset_name": "abstract_algebra"
 "description": "The following are questions (with answers) about abstract\
   \ algebra.\n\n"
-"tag": "mmlu_continuation_stem"
+"tag": "mmlu_stem_continuation"
 "include": "_continuation_template_yaml"
-"task": "mmlu_continuation_abstract_algebra"
+"task": "mmlu_abstract_algebra_continuation"

lm_eval/tasks/mmlu/continuation/mmlu_anatomy.yaml

 "dataset_name": "anatomy"
 "description": "The following are questions (with answers) about anatomy.\n\
   \n"
-"tag": "mmlu_continuation_stem"
+"tag": "mmlu_stem_continuation"
 "include": "_continuation_template_yaml"
-"task": "mmlu_continuation_anatomy"
+"task": "mmlu_anatomy_continuation"

lm_eval/tasks/mmlu/continuation/mmlu_astronomy.yaml

 "dataset_name": "astronomy"
 "description": "The following are questions (with answers) about astronomy.\n\
   \n"
-"tag": "mmlu_continuation_stem"
+"tag": "mmlu_stem_continuation"
 "include": "_continuation_template_yaml"
-"task": "mmlu_continuation_astronomy"
+"task": "mmlu_astronomy_continuation"

lm_eval/tasks/mmlu/continuation/mmlu_business_ethics.yaml

 "dataset_name": "business_ethics"
 "description": "The following are questions (with answers) about business\
   \ ethics.\n\n"
-"tag": "mmlu_continuation_other"
+"tag": "mmlu_other_continuation"
 "include": "_continuation_template_yaml"
-"task": "mmlu_continuation_business_ethics"
+"task": "mmlu_business_ethics_continuation"

lm_eval/tasks/mmlu/continuation/mmlu_clinical_knowledge.yaml

 "dataset_name": "clinical_knowledge"
 "description": "The following are questions (with answers) about clinical\
   \ knowledge.\n\n"
-"tag": "mmlu_continuation_other"
+"tag": "mmlu_other_continuation"
 "include": "_continuation_template_yaml"
-"task": "mmlu_continuation_clinical_knowledge"
+"task": "mmlu_clinical_knowledge_continuation"

lm_eval/tasks/mmlu/continuation/mmlu_college_biology.yaml

 "dataset_name": "college_biology"
 "description": "The following are questions (with answers) about college\
   \ biology.\n\n"
-"tag": "mmlu_continuation_stem"
+"tag": "mmlu_stem_continuation"
 "include": "_continuation_template_yaml"
-"task": "mmlu_continuation_college_biology"
+"task": "mmlu_college_biology_continuation"

lm_eval/tasks/mmlu/continuation/mmlu_college_chemistry.yaml

 "dataset_name": "college_chemistry"
 "description": "The following are questions (with answers) about college\
   \ chemistry.\n\n"
-"tag": "mmlu_continuation_stem"
+"tag": "mmlu_stem_continuation"
 "include": "_continuation_template_yaml"
-"task": "mmlu_continuation_college_chemistry"
+"task": "mmlu_college_chemistry_continuation"

lm_eval/tasks/mmlu/continuation/mmlu_college_computer_science.yaml

 "dataset_name": "college_computer_science"
 "description": "The following are questions (with answers) about college\
   \ computer science.\n\n"
-"tag": "mmlu_continuation_stem"
+"tag": "mmlu_stem_continuation"
 "include": "_continuation_template_yaml"
-"task": "mmlu_continuation_college_computer_science"
+"task": "mmlu_college_computer_science_continuation"

lm_eval/tasks/mmlu/continuation/mmlu_college_mathematics.yaml

 "dataset_name": "college_mathematics"
 "description": "The following are questions (with answers) about college\
   \ mathematics.\n\n"
-"tag": "mmlu_continuation_stem"
+"tag": "mmlu_stem_continuation"
 "include": "_continuation_template_yaml"
-"task": "mmlu_continuation_college_mathematics"
+"task": "mmlu_college_mathematics_continuation"

lm_eval/tasks/mmlu/continuation/mmlu_college_medicine.yaml

 "dataset_name": "college_medicine"
 "description": "The following are questions (with answers) about college\
   \ medicine.\n\n"
-"tag": "mmlu_continuation_other"
+"tag": "mmlu_other_continuation"
 "include": "_continuation_template_yaml"
-"task": "mmlu_continuation_college_medicine"
+"task": "mmlu_college_medicine_continuation"

lm_eval/tasks/mmlu/continuation/mmlu_college_physics.yaml

 "dataset_name": "college_physics"
 "description": "The following are questions (with answers) about college\
   \ physics.\n\n"
-"tag": "mmlu_continuation_stem"
+"tag": "mmlu_stem_continuation"
 "include": "_continuation_template_yaml"
-"task": "mmlu_continuation_college_physics"
+"task": "mmlu_college_physics_continuation"

lm_eval/tasks/mmlu/continuation/mmlu_computer_security.yaml

 "dataset_name": "computer_security"
 "description": "The following are questions (with answers) about computer\
   \ security.\n\n"
-"tag": "mmlu_continuation_stem"
+"tag": "mmlu_stem_continuation"
 "include": "_continuation_template_yaml"
-"task": "mmlu_continuation_computer_security"
+"task": "mmlu_computer_security_continuation"

lm_eval/tasks/mmlu/continuation/mmlu_conceptual_physics.yaml

 "dataset_name": "conceptual_physics"
 "description": "The following are questions (with answers) about conceptual\
   \ physics.\n\n"
-"tag": "mmlu_continuation_stem"
+"tag": "mmlu_stem_continuation"
 "include": "_continuation_template_yaml"
-"task": "mmlu_continuation_conceptual_physics"
+"task": "mmlu_conceptual_physics_continuation"

lm_eval/tasks/mmlu/continuation/mmlu_econometrics.yaml

 "dataset_name": "econometrics"
 "description": "The following are questions (with answers) about econometrics.\n\
   \n"
-"tag": "mmlu_continuation_social_sciences"
+"tag": "mmlu_social_sciences_continuation"
 "include": "_continuation_template_yaml"
-"task": "mmlu_continuation_econometrics"
+"task": "mmlu_econometrics_continuation"