Merge branch 'main' into mathvista

lm_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_other.yaml

+group: global_mmlu_full_ha_other
+task:
+  - global_mmlu_full_ha_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0

lm_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_social_sciences.yaml

+group: global_mmlu_full_ha_social_sciences
+task:
+  - global_mmlu_full_ha_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0

lm_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_stem.yaml

+group: global_mmlu_full_ha_stem
+task:
+  - global_mmlu_full_ha_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0

lm_eval/tasks/global_mmlu/full/ha/_ha_template_yaml

+dataset_path: CohereForAI/Global-MMLU
+dataset_name: ha
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0

lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_abstract_algebra.yaml

+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_abstract_algebra

lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_anatomy.yaml

+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_anatomy

lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_astronomy.yaml

+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_astronomy

lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_business_ethics.yaml

+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_ha_other_tasks
+task: global_mmlu_full_ha_business_ethics

lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_clinical_knowledge.yaml

+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_ha_other_tasks
+task: global_mmlu_full_ha_clinical_knowledge

lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_biology.yaml

+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_college_biology

lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_chemistry.yaml

+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_college_chemistry

lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_computer_science.yaml

+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_college_computer_science

lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_mathematics.yaml

+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_college_mathematics

lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_medicine.yaml

+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_ha_other_tasks
+task: global_mmlu_full_ha_college_medicine

lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_physics.yaml

+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_college_physics

lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_computer_security.yaml

+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_computer_security

lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_conceptual_physics.yaml

+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_conceptual_physics

lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_econometrics.yaml

+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_ha_social_sciences_tasks
+task: global_mmlu_full_ha_econometrics

lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_electrical_engineering.yaml

+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_electrical_engineering

lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_elementary_mathematics.yaml

+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_elementary_mathematics