From bbb8386ce239f964d85b15345127f18333a3f87c Mon Sep 17 00:00:00 2001 From: lintangsutawika Date: Tue, 16 Apr 2024 04:51:29 +0000 Subject: [PATCH] removed alt worlds prompts --- .../arc_challenge/README.md | 20 --- .../output_variation/_arc_challenge_alt_yaml | 23 ---- .../output_variation/arc_challenge_alt.yaml | 10 -- .../output_variation/style_01/a.yaml | 6 - .../output_variation/style_01/b.yaml | 6 - .../output_variation/style_01/c.yaml | 6 - .../output_variation/style_02/a.yaml | 6 - .../output_variation/style_02/b.yaml | 6 - .../output_variation/style_02/c.yaml | 6 - .../output_variation/style_03/a.yaml | 6 - .../output_variation/style_03/b.yaml | 6 - .../output_variation/style_03/c.yaml | 6 - .../output_variation/style_04/a.yaml | 6 - .../output_variation/style_04/b.yaml | 6 - .../output_variation/style_04/c.yaml | 6 - .../output_variation/style_05/a.yaml | 6 - .../output_variation/style_05/b.yaml | 6 - .../output_variation/style_05/c.yaml | 6 - .../output_variation/style_06/a.yaml | 6 - .../output_variation/style_06/b.yaml | 6 - .../output_variation/style_06/c.yaml | 6 - .../output_variation/style_07/a.yaml | 6 - .../output_variation/style_07/b.yaml | 6 - .../output_variation/style_07/c.yaml | 6 - .../output_variation/style_08/a.yaml | 6 - .../output_variation/style_08/b.yaml | 6 - .../output_variation/style_08/c.yaml | 6 - .../arc_challenge/output_variation/styles.py | 90 -------------- .../prompt_variation/_arc_challenge_alt_yaml | 21 ---- .../prompt_variation/style_01.yaml | 5 - .../prompt_variation/style_02.yaml | 5 - .../prompt_variation/style_03.yaml | 5 - .../arc/alternative_worlds/arc_easy/README.md | 20 --- .../output_variation/_arc_easy_alt_yaml | 24 ---- .../output_variation/arc_easy_alt.yaml | 10 -- .../arc_easy/output_variation/style_01/a.yaml | 6 - .../arc_easy/output_variation/style_01/b.yaml | 6 - .../arc_easy/output_variation/style_01/c.yaml | 6 - .../arc_easy/output_variation/style_02/a.yaml | 6 - .../arc_easy/output_variation/style_02/b.yaml | 6 - .../arc_easy/output_variation/style_02/c.yaml | 6 - .../arc_easy/output_variation/style_03/a.yaml | 6 - .../arc_easy/output_variation/style_03/b.yaml | 6 - .../arc_easy/output_variation/style_03/c.yaml | 6 - .../arc_easy/output_variation/style_04/a.yaml | 6 - .../arc_easy/output_variation/style_04/b.yaml | 6 - .../arc_easy/output_variation/style_04/c.yaml | 6 - .../arc_easy/output_variation/style_05/a.yaml | 6 - .../arc_easy/output_variation/style_05/b.yaml | 6 - .../arc_easy/output_variation/style_05/c.yaml | 6 - .../arc_easy/output_variation/style_06/a.yaml | 6 - .../arc_easy/output_variation/style_06/b.yaml | 6 - .../arc_easy/output_variation/style_06/c.yaml | 6 - .../arc_easy/output_variation/style_07/a.yaml | 6 - .../arc_easy/output_variation/style_07/b.yaml | 6 - .../arc_easy/output_variation/style_07/c.yaml | 6 - .../arc_easy/output_variation/style_08/a.yaml | 6 - .../arc_easy/output_variation/style_08/b.yaml | 6 - .../arc_easy/output_variation/style_08/c.yaml | 6 - .../arc_easy/output_variation/styles.py | 90 -------------- .../prompt_variation/_arc_easy_alt_yaml | 21 ---- .../arc_easy/prompt_variation/style_01.yaml | 5 - .../arc_easy/prompt_variation/style_02.yaml | 5 - .../arc_easy/prompt_variation/style_03.yaml | 5 - .../alternative_worlds/_template_yaml | 13 -- .../alternative_worlds/arithmetic_alt.yaml | 8 -- .../style_00/_template_00_yaml | 15 --- .../style_00/arithmetic_1dc.yaml | 4 - .../style_00/arithmetic_2da.yaml | 4 - .../style_00/arithmetic_2dm.yaml | 4 - .../style_00/arithmetic_2ds.yaml | 4 - .../style_00/arithmetic_3da.yaml | 4 - .../style_00/arithmetic_3ds.yaml | 4 - .../style_00/arithmetic_4da.yaml | 4 - .../style_00/arithmetic_4ds.yaml | 4 - .../style_00/arithmetic_5da.yaml | 4 - .../style_00/arithmetic_5ds.yaml | 4 - .../style_01/_template_01_yaml | 15 --- .../style_01/arithmetic_1dc.yaml | 4 - .../style_01/arithmetic_2da.yaml | 4 - .../style_01/arithmetic_2dm.yaml | 4 - .../style_01/arithmetic_2ds.yaml | 4 - .../style_01/arithmetic_3da.yaml | 4 - .../style_01/arithmetic_3ds.yaml | 4 - .../style_01/arithmetic_4da.yaml | 4 - .../style_01/arithmetic_4ds.yaml | 4 - .../style_01/arithmetic_5da.yaml | 4 - .../style_01/arithmetic_5ds.yaml | 4 - .../style_02/_template_02_yaml | 15 --- .../style_02/arithmetic_1dc.yaml | 4 - .../style_02/arithmetic_2da.yaml | 4 - .../style_02/arithmetic_2dm.yaml | 4 - .../style_02/arithmetic_2ds.yaml | 4 - .../style_02/arithmetic_3da.yaml | 4 - .../style_02/arithmetic_3ds.yaml | 4 - .../style_02/arithmetic_4da.yaml | 4 - .../style_02/arithmetic_4ds.yaml | 4 - .../style_02/arithmetic_5da.yaml | 4 - .../style_02/arithmetic_5ds.yaml | 4 - .../style_03/_template_03_yaml | 15 --- .../style_03/arithmetic_1dc.yaml | 4 - .../style_03/arithmetic_2da.yaml | 4 - .../style_03/arithmetic_2dm.yaml | 4 - .../style_03/arithmetic_2ds.yaml | 4 - .../style_03/arithmetic_3da.yaml | 4 - .../style_03/arithmetic_3ds.yaml | 4 - .../style_03/arithmetic_4da.yaml | 4 - .../style_03/arithmetic_4ds.yaml | 4 - .../style_03/arithmetic_5da.yaml | 4 - .../style_03/arithmetic_5ds.yaml | 4 - .../style_04/_template_04_yaml | 15 --- .../style_04/arithmetic_1dc.yaml | 4 - .../style_04/arithmetic_2da.yaml | 4 - .../style_04/arithmetic_2dm.yaml | 4 - .../style_04/arithmetic_2ds.yaml | 4 - .../style_04/arithmetic_3da.yaml | 4 - .../style_04/arithmetic_3ds.yaml | 4 - .../style_04/arithmetic_4da.yaml | 4 - .../style_04/arithmetic_4ds.yaml | 4 - .../style_04/arithmetic_5da.yaml | 4 - .../style_04/arithmetic_5ds.yaml | 4 - .../style_05/_template_05_yaml | 15 --- .../style_05/arithmetic_1dc.yaml | 4 - .../style_05/arithmetic_2da.yaml | 4 - .../style_05/arithmetic_2dm.yaml | 4 - .../style_05/arithmetic_2ds.yaml | 4 - .../style_05/arithmetic_3da.yaml | 4 - .../style_05/arithmetic_3ds.yaml | 4 - .../style_05/arithmetic_4da.yaml | 4 - .../style_05/arithmetic_4ds.yaml | 4 - .../style_05/arithmetic_5da.yaml | 4 - .../style_05/arithmetic_5ds.yaml | 4 - .../arithmetic/alternative_worlds/utils.py | 36 ------ .../tasks/bbh/alternative_worlds/README.md | 34 ------ .../prompt_variation/bbh_alt_pv_zeroshot.yaml | 5 - .../style_01/zeroshot/_zeroshot_template_yaml | 12 -- .../zeroshot/boolean_expressions.yaml | 6 - .../style_01/zeroshot/causal_judgement.yaml | 4 - .../style_01/zeroshot/date_understanding.yaml | 4 - .../style_01/zeroshot/disambiguation_qa.yaml | 4 - .../style_01/zeroshot/formal_fallacies.yaml | 6 - .../style_01/zeroshot/geometric_shapes.yaml | 4 - .../style_01/zeroshot/hyperbaton.yaml | 4 - .../logical_deduction_five_objects.yaml | 4 - .../logical_deduction_seven_objects.yaml | 4 - .../logical_deduction_three_objects.yaml | 4 - .../zeroshot/movie_recommendation.yaml | 5 - .../style_01/zeroshot/navigate.yaml | 4 - .../zeroshot/penguins_in_a_table.yaml | 4 - .../reasoning_about_colored_objects.yaml | 4 - .../style_01/zeroshot/ruin_names.yaml | 5 - .../salient_translation_error_detection.yaml | 4 - .../style_01/zeroshot/snarks.yaml | 4 - .../zeroshot/sports_understanding.yaml | 6 - .../style_01/zeroshot/temporal_sequences.yaml | 4 - ...racking_shuffled_objects_five_objects.yaml | 4 - ...acking_shuffled_objects_seven_objects.yaml | 4 - ...acking_shuffled_objects_three_objects.yaml | 4 - .../style_01/zeroshot/web_of_lies.yaml | 6 - .../style_02/zeroshot/_zeroshot_template_yaml | 12 -- .../zeroshot/boolean_expressions.yaml | 6 - .../style_02/zeroshot/causal_judgement.yaml | 4 - .../style_02/zeroshot/date_understanding.yaml | 4 - .../style_02/zeroshot/disambiguation_qa.yaml | 4 - .../style_02/zeroshot/formal_fallacies.yaml | 6 - .../style_02/zeroshot/geometric_shapes.yaml | 4 - .../style_02/zeroshot/hyperbaton.yaml | 4 - .../logical_deduction_five_objects.yaml | 4 - .../logical_deduction_seven_objects.yaml | 4 - .../logical_deduction_three_objects.yaml | 4 - .../zeroshot/movie_recommendation.yaml | 5 - .../style_02/zeroshot/navigate.yaml | 4 - .../zeroshot/penguins_in_a_table.yaml | 4 - .../reasoning_about_colored_objects.yaml | 4 - .../style_02/zeroshot/ruin_names.yaml | 5 - .../salient_translation_error_detection.yaml | 4 - .../style_02/zeroshot/snarks.yaml | 4 - .../zeroshot/sports_understanding.yaml | 6 - .../style_02/zeroshot/temporal_sequences.yaml | 4 - ...racking_shuffled_objects_five_objects.yaml | 4 - ...acking_shuffled_objects_seven_objects.yaml | 4 - ...acking_shuffled_objects_three_objects.yaml | 4 - .../style_02/zeroshot/web_of_lies.yaml | 6 - .../style_03/zeroshot/_zeroshot_template_yaml | 12 -- .../zeroshot/boolean_expressions.yaml | 6 - .../style_03/zeroshot/causal_judgement.yaml | 4 - .../style_03/zeroshot/date_understanding.yaml | 4 - .../style_03/zeroshot/disambiguation_qa.yaml | 4 - .../style_03/zeroshot/formal_fallacies.yaml | 6 - .../style_03/zeroshot/geometric_shapes.yaml | 4 - .../style_03/zeroshot/hyperbaton.yaml | 4 - .../logical_deduction_five_objects.yaml | 4 - .../logical_deduction_seven_objects.yaml | 4 - .../logical_deduction_three_objects.yaml | 4 - .../zeroshot/movie_recommendation.yaml | 5 - .../style_03/zeroshot/navigate.yaml | 4 - .../zeroshot/penguins_in_a_table.yaml | 4 - .../reasoning_about_colored_objects.yaml | 4 - .../style_03/zeroshot/ruin_names.yaml | 5 - .../salient_translation_error_detection.yaml | 4 - .../style_03/zeroshot/snarks.yaml | 4 - .../zeroshot/sports_understanding.yaml | 6 - .../style_03/zeroshot/temporal_sequences.yaml | 4 - ...racking_shuffled_objects_five_objects.yaml | 4 - ...acking_shuffled_objects_seven_objects.yaml | 4 - ...acking_shuffled_objects_three_objects.yaml | 4 - .../style_03/zeroshot/web_of_lies.yaml | 6 - .../prompt_variation/styles.py | 72 ----------- .../prompt_variation/utils.py | 31 ----- .../hellaswag/alternative_worlds/README.md | 20 --- .../alternative_worlds/_hellaswag_alt_yaml | 19 --- .../alternative_worlds/hellaswag_alt.yaml | 10 -- .../alternative_worlds/style_01/a.yaml | 7 -- .../alternative_worlds/style_01/b.yaml | 7 -- .../alternative_worlds/style_01/c.yaml | 7 -- .../alternative_worlds/style_02/a.yaml | 7 -- .../alternative_worlds/style_02/b.yaml | 7 -- .../alternative_worlds/style_02/c.yaml | 7 -- .../alternative_worlds/style_03/a.yaml | 7 -- .../alternative_worlds/style_03/b.yaml | 7 -- .../alternative_worlds/style_03/c.yaml | 7 -- .../alternative_worlds/style_04/a.yaml | 7 -- .../alternative_worlds/style_04/b.yaml | 7 -- .../alternative_worlds/style_04/c.yaml | 7 -- .../alternative_worlds/style_05/a.yaml | 7 -- .../alternative_worlds/style_05/b.yaml | 7 -- .../alternative_worlds/style_05/c.yaml | 7 -- .../alternative_worlds/style_06/a.yaml | 7 -- .../alternative_worlds/style_06/b.yaml | 7 -- .../alternative_worlds/style_06/c.yaml | 7 -- .../alternative_worlds/style_07/a.yaml | 7 -- .../alternative_worlds/style_07/b.yaml | 7 -- .../alternative_worlds/style_07/c.yaml | 7 -- .../alternative_worlds/style_08/a.yaml | 7 -- .../alternative_worlds/style_08/b.yaml | 7 -- .../alternative_worlds/style_08/c.yaml | 7 -- .../hellaswag/alternative_worlds/styles.py | 90 -------------- .../output_variation/README.md | 18 --- .../output_variation/_mathqa_alt_yaml | 10 -- .../output_variation/mathqa_alt.yaml | 10 -- .../output_variation/style_01/a.yaml | 5 - .../output_variation/style_01/b.yaml | 5 - .../output_variation/style_01/c.yaml | 5 - .../output_variation/style_02/a.yaml | 5 - .../output_variation/style_02/b.yaml | 5 - .../output_variation/style_02/c.yaml | 5 - .../output_variation/style_03/a.yaml | 5 - .../output_variation/style_03/b.yaml | 5 - .../output_variation/style_03/c.yaml | 5 - .../output_variation/style_04/a.yaml | 5 - .../output_variation/style_04/b.yaml | 5 - .../output_variation/style_04/c.yaml | 5 - .../output_variation/style_05/a.yaml | 5 - .../output_variation/style_05/b.yaml | 5 - .../output_variation/style_05/c.yaml | 5 - .../output_variation/style_06/a.yaml | 5 - .../output_variation/style_06/b.yaml | 5 - .../output_variation/style_06/c.yaml | 5 - .../output_variation/style_07/a.yaml | 5 - .../output_variation/style_07/b.yaml | 5 - .../output_variation/style_07/c.yaml | 5 - .../output_variation/style_08/a.yaml | 5 - .../output_variation/style_08/b.yaml | 5 - .../output_variation/style_08/c.yaml | 5 - .../output_variation/styles.py | 86 ------------- .../prompt_variation/style_01.yaml | 16 --- .../prompt_variation/style_02.yaml | 16 --- .../prompt_variation/style_03.yaml | 16 --- .../mmlu_output_variation.yaml | 10 -- .../mmlu_prompt_variation.yaml | 5 - .../style_01/_mmlu_ov_01.yaml | 5 - .../output_variation/style_01/a/_mmlu.yaml | 6 - .../style_01/a/_template_yaml | 11 -- .../style_01/a/mmlu_abstract_algebra.yaml | 6 - .../style_01/a/mmlu_anatomy.yaml | 6 - .../style_01/a/mmlu_astronomy.yaml | 6 - .../style_01/a/mmlu_business_ethics.yaml | 6 - .../style_01/a/mmlu_clinical_knowledge.yaml | 6 - .../style_01/a/mmlu_college_biology.yaml | 6 - .../style_01/a/mmlu_college_chemistry.yaml | 6 - .../a/mmlu_college_computer_science.yaml | 6 - .../style_01/a/mmlu_college_mathematics.yaml | 6 - .../style_01/a/mmlu_college_medicine.yaml | 6 - .../style_01/a/mmlu_college_physics.yaml | 6 - .../style_01/a/mmlu_computer_security.yaml | 6 - .../style_01/a/mmlu_conceptual_physics.yaml | 6 - .../style_01/a/mmlu_econometrics.yaml | 6 - .../a/mmlu_electrical_engineering.yaml | 6 - .../a/mmlu_elementary_mathematics.yaml | 6 - .../style_01/a/mmlu_formal_logic.yaml | 6 - .../style_01/a/mmlu_global_facts.yaml | 6 - .../style_01/a/mmlu_high_school_biology.yaml | 6 - .../a/mmlu_high_school_chemistry.yaml | 6 - .../a/mmlu_high_school_computer_science.yaml | 6 - .../a/mmlu_high_school_european_history.yaml | 6 - .../a/mmlu_high_school_geography.yaml | 6 - ...u_high_school_government_and_politics.yaml | 6 - .../a/mmlu_high_school_macroeconomics.yaml | 6 - .../a/mmlu_high_school_mathematics.yaml | 6 - .../a/mmlu_high_school_microeconomics.yaml | 6 - .../style_01/a/mmlu_high_school_physics.yaml | 6 - .../a/mmlu_high_school_psychology.yaml | 6 - .../a/mmlu_high_school_statistics.yaml | 6 - .../a/mmlu_high_school_us_history.yaml | 6 - .../a/mmlu_high_school_world_history.yaml | 6 - .../style_01/a/mmlu_human_aging.yaml | 6 - .../style_01/a/mmlu_human_sexuality.yaml | 6 - .../style_01/a/mmlu_international_law.yaml | 6 - .../style_01/a/mmlu_jurisprudence.yaml | 6 - .../style_01/a/mmlu_logical_fallacies.yaml | 6 - .../style_01/a/mmlu_machine_learning.yaml | 6 - .../style_01/a/mmlu_management.yaml | 6 - .../style_01/a/mmlu_marketing.yaml | 6 - .../style_01/a/mmlu_medical_genetics.yaml | 6 - .../style_01/a/mmlu_miscellaneous.yaml | 6 - .../style_01/a/mmlu_moral_disputes.yaml | 6 - .../style_01/a/mmlu_moral_scenarios.yaml | 6 - .../style_01/a/mmlu_nutrition.yaml | 6 - .../style_01/a/mmlu_philosophy.yaml | 6 - .../style_01/a/mmlu_prehistory.yaml | 6 - .../a/mmlu_professional_accounting.yaml | 6 - .../style_01/a/mmlu_professional_law.yaml | 6 - .../a/mmlu_professional_medicine.yaml | 6 - .../a/mmlu_professional_psychology.yaml | 6 - .../style_01/a/mmlu_public_relations.yaml | 6 - .../style_01/a/mmlu_security_studies.yaml | 6 - .../style_01/a/mmlu_sociology.yaml | 6 - .../style_01/a/mmlu_us_foreign_policy.yaml | 6 - .../style_01/a/mmlu_virology.yaml | 6 - .../style_01/a/mmlu_world_religions.yaml | 6 - .../output_variation/style_01/b/_mmlu.yaml | 6 - .../style_01/b/_template_yaml | 11 -- .../style_01/b/mmlu_abstract_algebra.yaml | 6 - .../style_01/b/mmlu_anatomy.yaml | 6 - .../style_01/b/mmlu_astronomy.yaml | 6 - .../style_01/b/mmlu_business_ethics.yaml | 6 - .../style_01/b/mmlu_clinical_knowledge.yaml | 6 - .../style_01/b/mmlu_college_biology.yaml | 6 - .../style_01/b/mmlu_college_chemistry.yaml | 6 - .../b/mmlu_college_computer_science.yaml | 6 - .../style_01/b/mmlu_college_mathematics.yaml | 6 - .../style_01/b/mmlu_college_medicine.yaml | 6 - .../style_01/b/mmlu_college_physics.yaml | 6 - .../style_01/b/mmlu_computer_security.yaml | 6 - .../style_01/b/mmlu_conceptual_physics.yaml | 6 - .../style_01/b/mmlu_econometrics.yaml | 6 - .../b/mmlu_electrical_engineering.yaml | 6 - .../b/mmlu_elementary_mathematics.yaml | 6 - .../style_01/b/mmlu_formal_logic.yaml | 6 - .../style_01/b/mmlu_global_facts.yaml | 6 - .../style_01/b/mmlu_high_school_biology.yaml | 6 - .../b/mmlu_high_school_chemistry.yaml | 6 - .../b/mmlu_high_school_computer_science.yaml | 6 - .../b/mmlu_high_school_european_history.yaml | 6 - .../b/mmlu_high_school_geography.yaml | 6 - ...u_high_school_government_and_politics.yaml | 6 - .../b/mmlu_high_school_macroeconomics.yaml | 6 - .../b/mmlu_high_school_mathematics.yaml | 6 - .../b/mmlu_high_school_microeconomics.yaml | 6 - .../style_01/b/mmlu_high_school_physics.yaml | 6 - .../b/mmlu_high_school_psychology.yaml | 6 - .../b/mmlu_high_school_statistics.yaml | 6 - .../b/mmlu_high_school_us_history.yaml | 6 - .../b/mmlu_high_school_world_history.yaml | 6 - .../style_01/b/mmlu_human_aging.yaml | 6 - .../style_01/b/mmlu_human_sexuality.yaml | 6 - .../style_01/b/mmlu_international_law.yaml | 6 - .../style_01/b/mmlu_jurisprudence.yaml | 6 - .../style_01/b/mmlu_logical_fallacies.yaml | 6 - .../style_01/b/mmlu_machine_learning.yaml | 6 - .../style_01/b/mmlu_management.yaml | 6 - .../style_01/b/mmlu_marketing.yaml | 6 - .../style_01/b/mmlu_medical_genetics.yaml | 6 - .../style_01/b/mmlu_miscellaneous.yaml | 6 - .../style_01/b/mmlu_moral_disputes.yaml | 6 - .../style_01/b/mmlu_moral_scenarios.yaml | 6 - .../style_01/b/mmlu_nutrition.yaml | 6 - .../style_01/b/mmlu_philosophy.yaml | 6 - .../style_01/b/mmlu_prehistory.yaml | 6 - .../b/mmlu_professional_accounting.yaml | 6 - .../style_01/b/mmlu_professional_law.yaml | 6 - .../b/mmlu_professional_medicine.yaml | 6 - .../b/mmlu_professional_psychology.yaml | 6 - .../style_01/b/mmlu_public_relations.yaml | 6 - .../style_01/b/mmlu_security_studies.yaml | 6 - .../style_01/b/mmlu_sociology.yaml | 6 - .../style_01/b/mmlu_us_foreign_policy.yaml | 6 - .../style_01/b/mmlu_virology.yaml | 6 - .../style_01/b/mmlu_world_religions.yaml | 6 - .../output_variation/style_01/c/_mmlu.yaml | 6 - .../style_01/c/_template_yaml | 11 -- .../style_01/c/mmlu_abstract_algebra.yaml | 6 - .../style_01/c/mmlu_anatomy.yaml | 6 - .../style_01/c/mmlu_astronomy.yaml | 6 - .../style_01/c/mmlu_business_ethics.yaml | 6 - .../style_01/c/mmlu_clinical_knowledge.yaml | 6 - .../style_01/c/mmlu_college_biology.yaml | 6 - .../style_01/c/mmlu_college_chemistry.yaml | 6 - .../c/mmlu_college_computer_science.yaml | 6 - .../style_01/c/mmlu_college_mathematics.yaml | 6 - .../style_01/c/mmlu_college_medicine.yaml | 6 - .../style_01/c/mmlu_college_physics.yaml | 6 - .../style_01/c/mmlu_computer_security.yaml | 6 - .../style_01/c/mmlu_conceptual_physics.yaml | 6 - .../style_01/c/mmlu_econometrics.yaml | 6 - .../c/mmlu_electrical_engineering.yaml | 6 - .../c/mmlu_elementary_mathematics.yaml | 6 - .../style_01/c/mmlu_formal_logic.yaml | 6 - .../style_01/c/mmlu_global_facts.yaml | 6 - .../style_01/c/mmlu_high_school_biology.yaml | 6 - .../c/mmlu_high_school_chemistry.yaml | 6 - .../c/mmlu_high_school_computer_science.yaml | 6 - .../c/mmlu_high_school_european_history.yaml | 6 - .../c/mmlu_high_school_geography.yaml | 6 - ...u_high_school_government_and_politics.yaml | 6 - .../c/mmlu_high_school_macroeconomics.yaml | 6 - .../c/mmlu_high_school_mathematics.yaml | 6 - .../c/mmlu_high_school_microeconomics.yaml | 6 - .../style_01/c/mmlu_high_school_physics.yaml | 6 - .../c/mmlu_high_school_psychology.yaml | 6 - .../c/mmlu_high_school_statistics.yaml | 6 - .../c/mmlu_high_school_us_history.yaml | 6 - .../c/mmlu_high_school_world_history.yaml | 6 - .../style_01/c/mmlu_human_aging.yaml | 6 - .../style_01/c/mmlu_human_sexuality.yaml | 6 - .../style_01/c/mmlu_international_law.yaml | 6 - .../style_01/c/mmlu_jurisprudence.yaml | 6 - .../style_01/c/mmlu_logical_fallacies.yaml | 6 - .../style_01/c/mmlu_machine_learning.yaml | 6 - .../style_01/c/mmlu_management.yaml | 6 - .../style_01/c/mmlu_marketing.yaml | 6 - .../style_01/c/mmlu_medical_genetics.yaml | 6 - .../style_01/c/mmlu_miscellaneous.yaml | 6 - .../style_01/c/mmlu_moral_disputes.yaml | 6 - .../style_01/c/mmlu_moral_scenarios.yaml | 6 - .../style_01/c/mmlu_nutrition.yaml | 6 - .../style_01/c/mmlu_philosophy.yaml | 6 - .../style_01/c/mmlu_prehistory.yaml | 6 - .../c/mmlu_professional_accounting.yaml | 6 - .../style_01/c/mmlu_professional_law.yaml | 6 - .../c/mmlu_professional_medicine.yaml | 6 - .../c/mmlu_professional_psychology.yaml | 6 - .../style_01/c/mmlu_public_relations.yaml | 6 - .../style_01/c/mmlu_security_studies.yaml | 6 - .../style_01/c/mmlu_sociology.yaml | 6 - .../style_01/c/mmlu_us_foreign_policy.yaml | 6 - .../style_01/c/mmlu_virology.yaml | 6 - .../style_01/c/mmlu_world_religions.yaml | 6 - .../style_02/_mmlu_ov_02.yaml | 5 - .../output_variation/style_02/a/_mmlu.yaml | 6 - .../style_02/a/_template_yaml | 11 -- .../style_02/a/mmlu_abstract_algebra.yaml | 6 - .../style_02/a/mmlu_anatomy.yaml | 6 - .../style_02/a/mmlu_astronomy.yaml | 6 - .../style_02/a/mmlu_business_ethics.yaml | 6 - .../style_02/a/mmlu_clinical_knowledge.yaml | 6 - .../style_02/a/mmlu_college_biology.yaml | 6 - .../style_02/a/mmlu_college_chemistry.yaml | 6 - .../a/mmlu_college_computer_science.yaml | 6 - .../style_02/a/mmlu_college_mathematics.yaml | 6 - .../style_02/a/mmlu_college_medicine.yaml | 6 - .../style_02/a/mmlu_college_physics.yaml | 6 - .../style_02/a/mmlu_computer_security.yaml | 6 - .../style_02/a/mmlu_conceptual_physics.yaml | 6 - .../style_02/a/mmlu_econometrics.yaml | 6 - .../a/mmlu_electrical_engineering.yaml | 6 - .../a/mmlu_elementary_mathematics.yaml | 6 - .../style_02/a/mmlu_formal_logic.yaml | 6 - .../style_02/a/mmlu_global_facts.yaml | 6 - .../style_02/a/mmlu_high_school_biology.yaml | 6 - .../a/mmlu_high_school_chemistry.yaml | 6 - .../a/mmlu_high_school_computer_science.yaml | 6 - .../a/mmlu_high_school_european_history.yaml | 6 - .../a/mmlu_high_school_geography.yaml | 6 - ...u_high_school_government_and_politics.yaml | 6 - .../a/mmlu_high_school_macroeconomics.yaml | 6 - .../a/mmlu_high_school_mathematics.yaml | 6 - .../a/mmlu_high_school_microeconomics.yaml | 6 - .../style_02/a/mmlu_high_school_physics.yaml | 6 - .../a/mmlu_high_school_psychology.yaml | 6 - .../a/mmlu_high_school_statistics.yaml | 6 - .../a/mmlu_high_school_us_history.yaml | 6 - .../a/mmlu_high_school_world_history.yaml | 6 - .../style_02/a/mmlu_human_aging.yaml | 6 - .../style_02/a/mmlu_human_sexuality.yaml | 6 - .../style_02/a/mmlu_international_law.yaml | 6 - .../style_02/a/mmlu_jurisprudence.yaml | 6 - .../style_02/a/mmlu_logical_fallacies.yaml | 6 - .../style_02/a/mmlu_machine_learning.yaml | 6 - .../style_02/a/mmlu_management.yaml | 6 - .../style_02/a/mmlu_marketing.yaml | 6 - .../style_02/a/mmlu_medical_genetics.yaml | 6 - .../style_02/a/mmlu_miscellaneous.yaml | 6 - .../style_02/a/mmlu_moral_disputes.yaml | 6 - .../style_02/a/mmlu_moral_scenarios.yaml | 6 - .../style_02/a/mmlu_nutrition.yaml | 6 - .../style_02/a/mmlu_philosophy.yaml | 6 - .../style_02/a/mmlu_prehistory.yaml | 6 - .../a/mmlu_professional_accounting.yaml | 6 - .../style_02/a/mmlu_professional_law.yaml | 6 - .../a/mmlu_professional_medicine.yaml | 6 - .../a/mmlu_professional_psychology.yaml | 6 - .../style_02/a/mmlu_public_relations.yaml | 6 - .../style_02/a/mmlu_security_studies.yaml | 6 - .../style_02/a/mmlu_sociology.yaml | 6 - .../style_02/a/mmlu_us_foreign_policy.yaml | 6 - .../style_02/a/mmlu_virology.yaml | 6 - .../style_02/a/mmlu_world_religions.yaml | 6 - .../output_variation/style_02/b/_mmlu.yaml | 6 - .../style_02/b/_template_yaml | 11 -- .../style_02/b/mmlu_abstract_algebra.yaml | 6 - .../style_02/b/mmlu_anatomy.yaml | 6 - .../style_02/b/mmlu_astronomy.yaml | 6 - .../style_02/b/mmlu_business_ethics.yaml | 6 - .../style_02/b/mmlu_clinical_knowledge.yaml | 6 - .../style_02/b/mmlu_college_biology.yaml | 6 - .../style_02/b/mmlu_college_chemistry.yaml | 6 - .../b/mmlu_college_computer_science.yaml | 6 - .../style_02/b/mmlu_college_mathematics.yaml | 6 - .../style_02/b/mmlu_college_medicine.yaml | 6 - .../style_02/b/mmlu_college_physics.yaml | 6 - .../style_02/b/mmlu_computer_security.yaml | 6 - .../style_02/b/mmlu_conceptual_physics.yaml | 6 - .../style_02/b/mmlu_econometrics.yaml | 6 - .../b/mmlu_electrical_engineering.yaml | 6 - .../b/mmlu_elementary_mathematics.yaml | 6 - .../style_02/b/mmlu_formal_logic.yaml | 6 - .../style_02/b/mmlu_global_facts.yaml | 6 - .../style_02/b/mmlu_high_school_biology.yaml | 6 - .../b/mmlu_high_school_chemistry.yaml | 6 - .../b/mmlu_high_school_computer_science.yaml | 6 - .../b/mmlu_high_school_european_history.yaml | 6 - .../b/mmlu_high_school_geography.yaml | 6 - ...u_high_school_government_and_politics.yaml | 6 - .../b/mmlu_high_school_macroeconomics.yaml | 6 - .../b/mmlu_high_school_mathematics.yaml | 6 - .../b/mmlu_high_school_microeconomics.yaml | 6 - .../style_02/b/mmlu_high_school_physics.yaml | 6 - .../b/mmlu_high_school_psychology.yaml | 6 - .../b/mmlu_high_school_statistics.yaml | 6 - .../b/mmlu_high_school_us_history.yaml | 6 - .../b/mmlu_high_school_world_history.yaml | 6 - .../style_02/b/mmlu_human_aging.yaml | 6 - .../style_02/b/mmlu_human_sexuality.yaml | 6 - .../style_02/b/mmlu_international_law.yaml | 6 - .../style_02/b/mmlu_jurisprudence.yaml | 6 - .../style_02/b/mmlu_logical_fallacies.yaml | 6 - .../style_02/b/mmlu_machine_learning.yaml | 6 - .../style_02/b/mmlu_management.yaml | 6 - .../style_02/b/mmlu_marketing.yaml | 6 - .../style_02/b/mmlu_medical_genetics.yaml | 6 - .../style_02/b/mmlu_miscellaneous.yaml | 6 - .../style_02/b/mmlu_moral_disputes.yaml | 6 - .../style_02/b/mmlu_moral_scenarios.yaml | 6 - .../style_02/b/mmlu_nutrition.yaml | 6 - .../style_02/b/mmlu_philosophy.yaml | 6 - .../style_02/b/mmlu_prehistory.yaml | 6 - .../b/mmlu_professional_accounting.yaml | 6 - .../style_02/b/mmlu_professional_law.yaml | 6 - .../b/mmlu_professional_medicine.yaml | 6 - .../b/mmlu_professional_psychology.yaml | 6 - .../style_02/b/mmlu_public_relations.yaml | 6 - .../style_02/b/mmlu_security_studies.yaml | 6 - .../style_02/b/mmlu_sociology.yaml | 6 - .../style_02/b/mmlu_us_foreign_policy.yaml | 6 - .../style_02/b/mmlu_virology.yaml | 6 - .../style_02/b/mmlu_world_religions.yaml | 6 - .../output_variation/style_02/c/_mmlu.yaml | 6 - .../style_02/c/_template_yaml | 11 -- .../style_02/c/mmlu_abstract_algebra.yaml | 6 - .../style_02/c/mmlu_anatomy.yaml | 6 - .../style_02/c/mmlu_astronomy.yaml | 6 - .../style_02/c/mmlu_business_ethics.yaml | 6 - .../style_02/c/mmlu_clinical_knowledge.yaml | 6 - .../style_02/c/mmlu_college_biology.yaml | 6 - .../style_02/c/mmlu_college_chemistry.yaml | 6 - .../c/mmlu_college_computer_science.yaml | 6 - .../style_02/c/mmlu_college_mathematics.yaml | 6 - .../style_02/c/mmlu_college_medicine.yaml | 6 - .../style_02/c/mmlu_college_physics.yaml | 6 - .../style_02/c/mmlu_computer_security.yaml | 6 - .../style_02/c/mmlu_conceptual_physics.yaml | 6 - .../style_02/c/mmlu_econometrics.yaml | 6 - .../c/mmlu_electrical_engineering.yaml | 6 - .../c/mmlu_elementary_mathematics.yaml | 6 - .../style_02/c/mmlu_formal_logic.yaml | 6 - .../style_02/c/mmlu_global_facts.yaml | 6 - .../style_02/c/mmlu_high_school_biology.yaml | 6 - .../c/mmlu_high_school_chemistry.yaml | 6 - .../c/mmlu_high_school_computer_science.yaml | 6 - .../c/mmlu_high_school_european_history.yaml | 6 - .../c/mmlu_high_school_geography.yaml | 6 - ...u_high_school_government_and_politics.yaml | 6 - .../c/mmlu_high_school_macroeconomics.yaml | 6 - .../c/mmlu_high_school_mathematics.yaml | 6 - .../c/mmlu_high_school_microeconomics.yaml | 6 - .../style_02/c/mmlu_high_school_physics.yaml | 6 - .../c/mmlu_high_school_psychology.yaml | 6 - .../c/mmlu_high_school_statistics.yaml | 6 - .../c/mmlu_high_school_us_history.yaml | 6 - .../c/mmlu_high_school_world_history.yaml | 6 - .../style_02/c/mmlu_human_aging.yaml | 6 - .../style_02/c/mmlu_human_sexuality.yaml | 6 - .../style_02/c/mmlu_international_law.yaml | 6 - .../style_02/c/mmlu_jurisprudence.yaml | 6 - .../style_02/c/mmlu_logical_fallacies.yaml | 6 - .../style_02/c/mmlu_machine_learning.yaml | 6 - .../style_02/c/mmlu_management.yaml | 6 - .../style_02/c/mmlu_marketing.yaml | 6 - .../style_02/c/mmlu_medical_genetics.yaml | 6 - .../style_02/c/mmlu_miscellaneous.yaml | 6 - .../style_02/c/mmlu_moral_disputes.yaml | 6 - .../style_02/c/mmlu_moral_scenarios.yaml | 6 - .../style_02/c/mmlu_nutrition.yaml | 6 - .../style_02/c/mmlu_philosophy.yaml | 6 - .../style_02/c/mmlu_prehistory.yaml | 6 - .../c/mmlu_professional_accounting.yaml | 6 - .../style_02/c/mmlu_professional_law.yaml | 6 - .../c/mmlu_professional_medicine.yaml | 6 - .../c/mmlu_professional_psychology.yaml | 6 - .../style_02/c/mmlu_public_relations.yaml | 6 - .../style_02/c/mmlu_security_studies.yaml | 6 - .../style_02/c/mmlu_sociology.yaml | 6 - .../style_02/c/mmlu_us_foreign_policy.yaml | 6 - .../style_02/c/mmlu_virology.yaml | 6 - .../style_02/c/mmlu_world_religions.yaml | 6 - .../style_03/_mmlu_ov_03.yaml | 5 - .../output_variation/style_03/a/_mmlu.yaml | 6 - .../style_03/a/_template_yaml | 11 -- .../style_03/a/mmlu_abstract_algebra.yaml | 6 - .../style_03/a/mmlu_anatomy.yaml | 6 - .../style_03/a/mmlu_astronomy.yaml | 6 - .../style_03/a/mmlu_business_ethics.yaml | 6 - .../style_03/a/mmlu_clinical_knowledge.yaml | 6 - .../style_03/a/mmlu_college_biology.yaml | 6 - .../style_03/a/mmlu_college_chemistry.yaml | 6 - .../a/mmlu_college_computer_science.yaml | 6 - .../style_03/a/mmlu_college_mathematics.yaml | 6 - .../style_03/a/mmlu_college_medicine.yaml | 6 - .../style_03/a/mmlu_college_physics.yaml | 6 - .../style_03/a/mmlu_computer_security.yaml | 6 - .../style_03/a/mmlu_conceptual_physics.yaml | 6 - .../style_03/a/mmlu_econometrics.yaml | 6 - .../a/mmlu_electrical_engineering.yaml | 6 - .../a/mmlu_elementary_mathematics.yaml | 6 - .../style_03/a/mmlu_formal_logic.yaml | 6 - .../style_03/a/mmlu_global_facts.yaml | 6 - .../style_03/a/mmlu_high_school_biology.yaml | 6 - .../a/mmlu_high_school_chemistry.yaml | 6 - .../a/mmlu_high_school_computer_science.yaml | 6 - .../a/mmlu_high_school_european_history.yaml | 6 - .../a/mmlu_high_school_geography.yaml | 6 - ...u_high_school_government_and_politics.yaml | 6 - .../a/mmlu_high_school_macroeconomics.yaml | 6 - .../a/mmlu_high_school_mathematics.yaml | 6 - .../a/mmlu_high_school_microeconomics.yaml | 6 - .../style_03/a/mmlu_high_school_physics.yaml | 6 - .../a/mmlu_high_school_psychology.yaml | 6 - .../a/mmlu_high_school_statistics.yaml | 6 - .../a/mmlu_high_school_us_history.yaml | 6 - .../a/mmlu_high_school_world_history.yaml | 6 - .../style_03/a/mmlu_human_aging.yaml | 6 - .../style_03/a/mmlu_human_sexuality.yaml | 6 - .../style_03/a/mmlu_international_law.yaml | 6 - .../style_03/a/mmlu_jurisprudence.yaml | 6 - .../style_03/a/mmlu_logical_fallacies.yaml | 6 - .../style_03/a/mmlu_machine_learning.yaml | 6 - .../style_03/a/mmlu_management.yaml | 6 - .../style_03/a/mmlu_marketing.yaml | 6 - .../style_03/a/mmlu_medical_genetics.yaml | 6 - .../style_03/a/mmlu_miscellaneous.yaml | 6 - .../style_03/a/mmlu_moral_disputes.yaml | 6 - .../style_03/a/mmlu_moral_scenarios.yaml | 6 - .../style_03/a/mmlu_nutrition.yaml | 6 - .../style_03/a/mmlu_philosophy.yaml | 6 - .../style_03/a/mmlu_prehistory.yaml | 6 - .../a/mmlu_professional_accounting.yaml | 6 - .../style_03/a/mmlu_professional_law.yaml | 6 - .../a/mmlu_professional_medicine.yaml | 6 - .../a/mmlu_professional_psychology.yaml | 6 - .../style_03/a/mmlu_public_relations.yaml | 6 - .../style_03/a/mmlu_security_studies.yaml | 6 - .../style_03/a/mmlu_sociology.yaml | 6 - .../style_03/a/mmlu_us_foreign_policy.yaml | 6 - .../style_03/a/mmlu_virology.yaml | 6 - .../style_03/a/mmlu_world_religions.yaml | 6 - .../output_variation/style_03/b/_mmlu.yaml | 6 - .../style_03/b/_template_yaml | 11 -- .../style_03/b/mmlu_abstract_algebra.yaml | 6 - .../style_03/b/mmlu_anatomy.yaml | 6 - .../style_03/b/mmlu_astronomy.yaml | 6 - .../style_03/b/mmlu_business_ethics.yaml | 6 - .../style_03/b/mmlu_clinical_knowledge.yaml | 6 - .../style_03/b/mmlu_college_biology.yaml | 6 - .../style_03/b/mmlu_college_chemistry.yaml | 6 - .../b/mmlu_college_computer_science.yaml | 6 - .../style_03/b/mmlu_college_mathematics.yaml | 6 - .../style_03/b/mmlu_college_medicine.yaml | 6 - .../style_03/b/mmlu_college_physics.yaml | 6 - .../style_03/b/mmlu_computer_security.yaml | 6 - .../style_03/b/mmlu_conceptual_physics.yaml | 6 - .../style_03/b/mmlu_econometrics.yaml | 6 - .../b/mmlu_electrical_engineering.yaml | 6 - .../b/mmlu_elementary_mathematics.yaml | 6 - .../style_03/b/mmlu_formal_logic.yaml | 6 - .../style_03/b/mmlu_global_facts.yaml | 6 - .../style_03/b/mmlu_high_school_biology.yaml | 6 - .../b/mmlu_high_school_chemistry.yaml | 6 - .../b/mmlu_high_school_computer_science.yaml | 6 - .../b/mmlu_high_school_european_history.yaml | 6 - .../b/mmlu_high_school_geography.yaml | 6 - ...u_high_school_government_and_politics.yaml | 6 - .../b/mmlu_high_school_macroeconomics.yaml | 6 - .../b/mmlu_high_school_mathematics.yaml | 6 - .../b/mmlu_high_school_microeconomics.yaml | 6 - .../style_03/b/mmlu_high_school_physics.yaml | 6 - .../b/mmlu_high_school_psychology.yaml | 6 - .../b/mmlu_high_school_statistics.yaml | 6 - .../b/mmlu_high_school_us_history.yaml | 6 - .../b/mmlu_high_school_world_history.yaml | 6 - .../style_03/b/mmlu_human_aging.yaml | 6 - .../style_03/b/mmlu_human_sexuality.yaml | 6 - .../style_03/b/mmlu_international_law.yaml | 6 - .../style_03/b/mmlu_jurisprudence.yaml | 6 - .../style_03/b/mmlu_logical_fallacies.yaml | 6 - .../style_03/b/mmlu_machine_learning.yaml | 6 - .../style_03/b/mmlu_management.yaml | 6 - .../style_03/b/mmlu_marketing.yaml | 6 - .../style_03/b/mmlu_medical_genetics.yaml | 6 - .../style_03/b/mmlu_miscellaneous.yaml | 6 - .../style_03/b/mmlu_moral_disputes.yaml | 6 - .../style_03/b/mmlu_moral_scenarios.yaml | 6 - .../style_03/b/mmlu_nutrition.yaml | 6 - .../style_03/b/mmlu_philosophy.yaml | 6 - .../style_03/b/mmlu_prehistory.yaml | 6 - .../b/mmlu_professional_accounting.yaml | 6 - .../style_03/b/mmlu_professional_law.yaml | 6 - .../b/mmlu_professional_medicine.yaml | 6 - .../b/mmlu_professional_psychology.yaml | 6 - .../style_03/b/mmlu_public_relations.yaml | 6 - .../style_03/b/mmlu_security_studies.yaml | 6 - .../style_03/b/mmlu_sociology.yaml | 6 - .../style_03/b/mmlu_us_foreign_policy.yaml | 6 - .../style_03/b/mmlu_virology.yaml | 6 - .../style_03/b/mmlu_world_religions.yaml | 6 - .../output_variation/style_03/c/_mmlu.yaml | 6 - .../style_03/c/_template_yaml | 11 -- .../style_03/c/mmlu_abstract_algebra.yaml | 6 - .../style_03/c/mmlu_anatomy.yaml | 6 - .../style_03/c/mmlu_astronomy.yaml | 6 - .../style_03/c/mmlu_business_ethics.yaml | 6 - .../style_03/c/mmlu_clinical_knowledge.yaml | 6 - .../style_03/c/mmlu_college_biology.yaml | 6 - .../style_03/c/mmlu_college_chemistry.yaml | 6 - .../c/mmlu_college_computer_science.yaml | 6 - .../style_03/c/mmlu_college_mathematics.yaml | 6 - .../style_03/c/mmlu_college_medicine.yaml | 6 - .../style_03/c/mmlu_college_physics.yaml | 6 - .../style_03/c/mmlu_computer_security.yaml | 6 - .../style_03/c/mmlu_conceptual_physics.yaml | 6 - .../style_03/c/mmlu_econometrics.yaml | 6 - .../c/mmlu_electrical_engineering.yaml | 6 - .../c/mmlu_elementary_mathematics.yaml | 6 - .../style_03/c/mmlu_formal_logic.yaml | 6 - .../style_03/c/mmlu_global_facts.yaml | 6 - .../style_03/c/mmlu_high_school_biology.yaml | 6 - .../c/mmlu_high_school_chemistry.yaml | 6 - .../c/mmlu_high_school_computer_science.yaml | 6 - .../c/mmlu_high_school_european_history.yaml | 6 - .../c/mmlu_high_school_geography.yaml | 6 - ...u_high_school_government_and_politics.yaml | 6 - .../c/mmlu_high_school_macroeconomics.yaml | 6 - .../c/mmlu_high_school_mathematics.yaml | 6 - .../c/mmlu_high_school_microeconomics.yaml | 6 - .../style_03/c/mmlu_high_school_physics.yaml | 6 - .../c/mmlu_high_school_psychology.yaml | 6 - .../c/mmlu_high_school_statistics.yaml | 6 - .../c/mmlu_high_school_us_history.yaml | 6 - .../c/mmlu_high_school_world_history.yaml | 6 - .../style_03/c/mmlu_human_aging.yaml | 6 - .../style_03/c/mmlu_human_sexuality.yaml | 6 - .../style_03/c/mmlu_international_law.yaml | 6 - .../style_03/c/mmlu_jurisprudence.yaml | 6 - .../style_03/c/mmlu_logical_fallacies.yaml | 6 - .../style_03/c/mmlu_machine_learning.yaml | 6 - .../style_03/c/mmlu_management.yaml | 6 - .../style_03/c/mmlu_marketing.yaml | 6 - .../style_03/c/mmlu_medical_genetics.yaml | 6 - .../style_03/c/mmlu_miscellaneous.yaml | 6 - .../style_03/c/mmlu_moral_disputes.yaml | 6 - .../style_03/c/mmlu_moral_scenarios.yaml | 6 - .../style_03/c/mmlu_nutrition.yaml | 6 - .../style_03/c/mmlu_philosophy.yaml | 6 - .../style_03/c/mmlu_prehistory.yaml | 6 - .../c/mmlu_professional_accounting.yaml | 6 - .../style_03/c/mmlu_professional_law.yaml | 6 - .../c/mmlu_professional_medicine.yaml | 6 - .../c/mmlu_professional_psychology.yaml | 6 - .../style_03/c/mmlu_public_relations.yaml | 6 - .../style_03/c/mmlu_security_studies.yaml | 6 - .../style_03/c/mmlu_sociology.yaml | 6 - .../style_03/c/mmlu_us_foreign_policy.yaml | 6 - .../style_03/c/mmlu_virology.yaml | 6 - .../style_03/c/mmlu_world_religions.yaml | 6 - .../style_04/_mmlu_ov_04.yaml | 5 - .../output_variation/style_04/a/_mmlu.yaml | 6 - .../style_04/a/_template_yaml | 11 -- .../style_04/a/mmlu_abstract_algebra.yaml | 6 - .../style_04/a/mmlu_anatomy.yaml | 6 - .../style_04/a/mmlu_astronomy.yaml | 6 - .../style_04/a/mmlu_business_ethics.yaml | 6 - .../style_04/a/mmlu_clinical_knowledge.yaml | 6 - .../style_04/a/mmlu_college_biology.yaml | 6 - .../style_04/a/mmlu_college_chemistry.yaml | 6 - .../a/mmlu_college_computer_science.yaml | 6 - .../style_04/a/mmlu_college_mathematics.yaml | 6 - .../style_04/a/mmlu_college_medicine.yaml | 6 - .../style_04/a/mmlu_college_physics.yaml | 6 - .../style_04/a/mmlu_computer_security.yaml | 6 - .../style_04/a/mmlu_conceptual_physics.yaml | 6 - .../style_04/a/mmlu_econometrics.yaml | 6 - .../a/mmlu_electrical_engineering.yaml | 6 - .../a/mmlu_elementary_mathematics.yaml | 6 - .../style_04/a/mmlu_formal_logic.yaml | 6 - .../style_04/a/mmlu_global_facts.yaml | 6 - .../style_04/a/mmlu_high_school_biology.yaml | 6 - .../a/mmlu_high_school_chemistry.yaml | 6 - .../a/mmlu_high_school_computer_science.yaml | 6 - .../a/mmlu_high_school_european_history.yaml | 6 - .../a/mmlu_high_school_geography.yaml | 6 - ...u_high_school_government_and_politics.yaml | 6 - .../a/mmlu_high_school_macroeconomics.yaml | 6 - .../a/mmlu_high_school_mathematics.yaml | 6 - .../a/mmlu_high_school_microeconomics.yaml | 6 - .../style_04/a/mmlu_high_school_physics.yaml | 6 - .../a/mmlu_high_school_psychology.yaml | 6 - .../a/mmlu_high_school_statistics.yaml | 6 - .../a/mmlu_high_school_us_history.yaml | 6 - .../a/mmlu_high_school_world_history.yaml | 6 - .../style_04/a/mmlu_human_aging.yaml | 6 - .../style_04/a/mmlu_human_sexuality.yaml | 6 - .../style_04/a/mmlu_international_law.yaml | 6 - .../style_04/a/mmlu_jurisprudence.yaml | 6 - .../style_04/a/mmlu_logical_fallacies.yaml | 6 - .../style_04/a/mmlu_machine_learning.yaml | 6 - .../style_04/a/mmlu_management.yaml | 6 - .../style_04/a/mmlu_marketing.yaml | 6 - .../style_04/a/mmlu_medical_genetics.yaml | 6 - .../style_04/a/mmlu_miscellaneous.yaml | 6 - .../style_04/a/mmlu_moral_disputes.yaml | 6 - .../style_04/a/mmlu_moral_scenarios.yaml | 6 - .../style_04/a/mmlu_nutrition.yaml | 6 - .../style_04/a/mmlu_philosophy.yaml | 6 - .../style_04/a/mmlu_prehistory.yaml | 6 - .../a/mmlu_professional_accounting.yaml | 6 - .../style_04/a/mmlu_professional_law.yaml | 6 - .../a/mmlu_professional_medicine.yaml | 6 - .../a/mmlu_professional_psychology.yaml | 6 - .../style_04/a/mmlu_public_relations.yaml | 6 - .../style_04/a/mmlu_security_studies.yaml | 6 - .../style_04/a/mmlu_sociology.yaml | 6 - .../style_04/a/mmlu_us_foreign_policy.yaml | 6 - .../style_04/a/mmlu_virology.yaml | 6 - .../style_04/a/mmlu_world_religions.yaml | 6 - .../output_variation/style_04/b/_mmlu.yaml | 6 - .../style_04/b/_template_yaml | 11 -- .../style_04/b/mmlu_abstract_algebra.yaml | 6 - .../style_04/b/mmlu_anatomy.yaml | 6 - .../style_04/b/mmlu_astronomy.yaml | 6 - .../style_04/b/mmlu_business_ethics.yaml | 6 - .../style_04/b/mmlu_clinical_knowledge.yaml | 6 - .../style_04/b/mmlu_college_biology.yaml | 6 - .../style_04/b/mmlu_college_chemistry.yaml | 6 - .../b/mmlu_college_computer_science.yaml | 6 - .../style_04/b/mmlu_college_mathematics.yaml | 6 - .../style_04/b/mmlu_college_medicine.yaml | 6 - .../style_04/b/mmlu_college_physics.yaml | 6 - .../style_04/b/mmlu_computer_security.yaml | 6 - .../style_04/b/mmlu_conceptual_physics.yaml | 6 - .../style_04/b/mmlu_econometrics.yaml | 6 - .../b/mmlu_electrical_engineering.yaml | 6 - .../b/mmlu_elementary_mathematics.yaml | 6 - .../style_04/b/mmlu_formal_logic.yaml | 6 - .../style_04/b/mmlu_global_facts.yaml | 6 - .../style_04/b/mmlu_high_school_biology.yaml | 6 - .../b/mmlu_high_school_chemistry.yaml | 6 - .../b/mmlu_high_school_computer_science.yaml | 6 - .../b/mmlu_high_school_european_history.yaml | 6 - .../b/mmlu_high_school_geography.yaml | 6 - ...u_high_school_government_and_politics.yaml | 6 - .../b/mmlu_high_school_macroeconomics.yaml | 6 - .../b/mmlu_high_school_mathematics.yaml | 6 - .../b/mmlu_high_school_microeconomics.yaml | 6 - .../style_04/b/mmlu_high_school_physics.yaml | 6 - .../b/mmlu_high_school_psychology.yaml | 6 - .../b/mmlu_high_school_statistics.yaml | 6 - .../b/mmlu_high_school_us_history.yaml | 6 - .../b/mmlu_high_school_world_history.yaml | 6 - .../style_04/b/mmlu_human_aging.yaml | 6 - .../style_04/b/mmlu_human_sexuality.yaml | 6 - .../style_04/b/mmlu_international_law.yaml | 6 - .../style_04/b/mmlu_jurisprudence.yaml | 6 - .../style_04/b/mmlu_logical_fallacies.yaml | 6 - .../style_04/b/mmlu_machine_learning.yaml | 6 - .../style_04/b/mmlu_management.yaml | 6 - .../style_04/b/mmlu_marketing.yaml | 6 - .../style_04/b/mmlu_medical_genetics.yaml | 6 - .../style_04/b/mmlu_miscellaneous.yaml | 6 - .../style_04/b/mmlu_moral_disputes.yaml | 6 - .../style_04/b/mmlu_moral_scenarios.yaml | 6 - .../style_04/b/mmlu_nutrition.yaml | 6 - .../style_04/b/mmlu_philosophy.yaml | 6 - .../style_04/b/mmlu_prehistory.yaml | 6 - .../b/mmlu_professional_accounting.yaml | 6 - .../style_04/b/mmlu_professional_law.yaml | 6 - .../b/mmlu_professional_medicine.yaml | 6 - .../b/mmlu_professional_psychology.yaml | 6 - .../style_04/b/mmlu_public_relations.yaml | 6 - .../style_04/b/mmlu_security_studies.yaml | 6 - .../style_04/b/mmlu_sociology.yaml | 6 - .../style_04/b/mmlu_us_foreign_policy.yaml | 6 - .../style_04/b/mmlu_virology.yaml | 6 - .../style_04/b/mmlu_world_religions.yaml | 6 - .../output_variation/style_04/c/_mmlu.yaml | 6 - .../style_04/c/_template_yaml | 11 -- .../style_04/c/mmlu_abstract_algebra.yaml | 6 - .../style_04/c/mmlu_anatomy.yaml | 6 - .../style_04/c/mmlu_astronomy.yaml | 6 - .../style_04/c/mmlu_business_ethics.yaml | 6 - .../style_04/c/mmlu_clinical_knowledge.yaml | 6 - .../style_04/c/mmlu_college_biology.yaml | 6 - .../style_04/c/mmlu_college_chemistry.yaml | 6 - .../c/mmlu_college_computer_science.yaml | 6 - .../style_04/c/mmlu_college_mathematics.yaml | 6 - .../style_04/c/mmlu_college_medicine.yaml | 6 - .../style_04/c/mmlu_college_physics.yaml | 6 - .../style_04/c/mmlu_computer_security.yaml | 6 - .../style_04/c/mmlu_conceptual_physics.yaml | 6 - .../style_04/c/mmlu_econometrics.yaml | 6 - .../c/mmlu_electrical_engineering.yaml | 6 - .../c/mmlu_elementary_mathematics.yaml | 6 - .../style_04/c/mmlu_formal_logic.yaml | 6 - .../style_04/c/mmlu_global_facts.yaml | 6 - .../style_04/c/mmlu_high_school_biology.yaml | 6 - .../c/mmlu_high_school_chemistry.yaml | 6 - .../c/mmlu_high_school_computer_science.yaml | 6 - .../c/mmlu_high_school_european_history.yaml | 6 - .../c/mmlu_high_school_geography.yaml | 6 - ...u_high_school_government_and_politics.yaml | 6 - .../c/mmlu_high_school_macroeconomics.yaml | 6 - .../c/mmlu_high_school_mathematics.yaml | 6 - .../c/mmlu_high_school_microeconomics.yaml | 6 - .../style_04/c/mmlu_high_school_physics.yaml | 6 - .../c/mmlu_high_school_psychology.yaml | 6 - .../c/mmlu_high_school_statistics.yaml | 6 - .../c/mmlu_high_school_us_history.yaml | 6 - .../c/mmlu_high_school_world_history.yaml | 6 - .../style_04/c/mmlu_human_aging.yaml | 6 - .../style_04/c/mmlu_human_sexuality.yaml | 6 - .../style_04/c/mmlu_international_law.yaml | 6 - .../style_04/c/mmlu_jurisprudence.yaml | 6 - .../style_04/c/mmlu_logical_fallacies.yaml | 6 - .../style_04/c/mmlu_machine_learning.yaml | 6 - .../style_04/c/mmlu_management.yaml | 6 - .../style_04/c/mmlu_marketing.yaml | 6 - .../style_04/c/mmlu_medical_genetics.yaml | 6 - .../style_04/c/mmlu_miscellaneous.yaml | 6 - .../style_04/c/mmlu_moral_disputes.yaml | 6 - .../style_04/c/mmlu_moral_scenarios.yaml | 6 - .../style_04/c/mmlu_nutrition.yaml | 6 - .../style_04/c/mmlu_philosophy.yaml | 6 - .../style_04/c/mmlu_prehistory.yaml | 6 - .../c/mmlu_professional_accounting.yaml | 6 - .../style_04/c/mmlu_professional_law.yaml | 6 - .../c/mmlu_professional_medicine.yaml | 6 - .../c/mmlu_professional_psychology.yaml | 6 - .../style_04/c/mmlu_public_relations.yaml | 6 - .../style_04/c/mmlu_security_studies.yaml | 6 - .../style_04/c/mmlu_sociology.yaml | 6 - .../style_04/c/mmlu_us_foreign_policy.yaml | 6 - .../style_04/c/mmlu_virology.yaml | 6 - .../style_04/c/mmlu_world_religions.yaml | 6 - .../style_05/_mmlu_ov_05.yaml | 5 - .../output_variation/style_05/a/_mmlu.yaml | 6 - .../style_05/a/_template_yaml | 13 -- .../style_05/a/mmlu_abstract_algebra.yaml | 6 - .../style_05/a/mmlu_anatomy.yaml | 6 - .../style_05/a/mmlu_astronomy.yaml | 6 - .../style_05/a/mmlu_business_ethics.yaml | 6 - .../style_05/a/mmlu_clinical_knowledge.yaml | 6 - .../style_05/a/mmlu_college_biology.yaml | 6 - .../style_05/a/mmlu_college_chemistry.yaml | 6 - .../a/mmlu_college_computer_science.yaml | 6 - .../style_05/a/mmlu_college_mathematics.yaml | 6 - .../style_05/a/mmlu_college_medicine.yaml | 6 - .../style_05/a/mmlu_college_physics.yaml | 6 - .../style_05/a/mmlu_computer_security.yaml | 6 - .../style_05/a/mmlu_conceptual_physics.yaml | 6 - .../style_05/a/mmlu_econometrics.yaml | 6 - .../a/mmlu_electrical_engineering.yaml | 6 - .../a/mmlu_elementary_mathematics.yaml | 6 - .../style_05/a/mmlu_formal_logic.yaml | 6 - .../style_05/a/mmlu_global_facts.yaml | 6 - .../style_05/a/mmlu_high_school_biology.yaml | 6 - .../a/mmlu_high_school_chemistry.yaml | 6 - .../a/mmlu_high_school_computer_science.yaml | 6 - .../a/mmlu_high_school_european_history.yaml | 6 - .../a/mmlu_high_school_geography.yaml | 6 - ...u_high_school_government_and_politics.yaml | 6 - .../a/mmlu_high_school_macroeconomics.yaml | 6 - .../a/mmlu_high_school_mathematics.yaml | 6 - .../a/mmlu_high_school_microeconomics.yaml | 6 - .../style_05/a/mmlu_high_school_physics.yaml | 6 - .../a/mmlu_high_school_psychology.yaml | 6 - .../a/mmlu_high_school_statistics.yaml | 6 - .../a/mmlu_high_school_us_history.yaml | 6 - .../a/mmlu_high_school_world_history.yaml | 6 - .../style_05/a/mmlu_human_aging.yaml | 6 - .../style_05/a/mmlu_human_sexuality.yaml | 6 - .../style_05/a/mmlu_international_law.yaml | 6 - .../style_05/a/mmlu_jurisprudence.yaml | 6 - .../style_05/a/mmlu_logical_fallacies.yaml | 6 - .../style_05/a/mmlu_machine_learning.yaml | 6 - .../style_05/a/mmlu_management.yaml | 6 - .../style_05/a/mmlu_marketing.yaml | 6 - .../style_05/a/mmlu_medical_genetics.yaml | 6 - .../style_05/a/mmlu_miscellaneous.yaml | 6 - .../style_05/a/mmlu_moral_disputes.yaml | 6 - .../style_05/a/mmlu_moral_scenarios.yaml | 6 - .../style_05/a/mmlu_nutrition.yaml | 6 - .../style_05/a/mmlu_philosophy.yaml | 6 - .../style_05/a/mmlu_prehistory.yaml | 6 - .../a/mmlu_professional_accounting.yaml | 6 - .../style_05/a/mmlu_professional_law.yaml | 6 - .../a/mmlu_professional_medicine.yaml | 6 - .../a/mmlu_professional_psychology.yaml | 6 - .../style_05/a/mmlu_public_relations.yaml | 6 - .../style_05/a/mmlu_security_studies.yaml | 6 - .../style_05/a/mmlu_sociology.yaml | 6 - .../style_05/a/mmlu_us_foreign_policy.yaml | 6 - .../style_05/a/mmlu_virology.yaml | 6 - .../style_05/a/mmlu_world_religions.yaml | 6 - .../output_variation/style_05/b/_mmlu.yaml | 6 - .../style_05/b/_template_yaml | 11 -- .../style_05/b/mmlu_abstract_algebra.yaml | 6 - .../style_05/b/mmlu_anatomy.yaml | 6 - .../style_05/b/mmlu_astronomy.yaml | 6 - .../style_05/b/mmlu_business_ethics.yaml | 6 - .../style_05/b/mmlu_clinical_knowledge.yaml | 6 - .../style_05/b/mmlu_college_biology.yaml | 6 - .../style_05/b/mmlu_college_chemistry.yaml | 6 - .../b/mmlu_college_computer_science.yaml | 6 - .../style_05/b/mmlu_college_mathematics.yaml | 6 - .../style_05/b/mmlu_college_medicine.yaml | 6 - .../style_05/b/mmlu_college_physics.yaml | 6 - .../style_05/b/mmlu_computer_security.yaml | 6 - .../style_05/b/mmlu_conceptual_physics.yaml | 6 - .../style_05/b/mmlu_econometrics.yaml | 6 - .../b/mmlu_electrical_engineering.yaml | 6 - .../b/mmlu_elementary_mathematics.yaml | 6 - .../style_05/b/mmlu_formal_logic.yaml | 6 - .../style_05/b/mmlu_global_facts.yaml | 6 - .../style_05/b/mmlu_high_school_biology.yaml | 6 - .../b/mmlu_high_school_chemistry.yaml | 6 - .../b/mmlu_high_school_computer_science.yaml | 6 - .../b/mmlu_high_school_european_history.yaml | 6 - .../b/mmlu_high_school_geography.yaml | 6 - ...u_high_school_government_and_politics.yaml | 6 - .../b/mmlu_high_school_macroeconomics.yaml | 6 - .../b/mmlu_high_school_mathematics.yaml | 6 - .../b/mmlu_high_school_microeconomics.yaml | 6 - .../style_05/b/mmlu_high_school_physics.yaml | 6 - .../b/mmlu_high_school_psychology.yaml | 6 - .../b/mmlu_high_school_statistics.yaml | 6 - .../b/mmlu_high_school_us_history.yaml | 6 - .../b/mmlu_high_school_world_history.yaml | 6 - .../style_05/b/mmlu_human_aging.yaml | 6 - .../style_05/b/mmlu_human_sexuality.yaml | 6 - .../style_05/b/mmlu_international_law.yaml | 6 - .../style_05/b/mmlu_jurisprudence.yaml | 6 - .../style_05/b/mmlu_logical_fallacies.yaml | 6 - .../style_05/b/mmlu_machine_learning.yaml | 6 - .../style_05/b/mmlu_management.yaml | 6 - .../style_05/b/mmlu_marketing.yaml | 6 - .../style_05/b/mmlu_medical_genetics.yaml | 6 - .../style_05/b/mmlu_miscellaneous.yaml | 6 - .../style_05/b/mmlu_moral_disputes.yaml | 6 - .../style_05/b/mmlu_moral_scenarios.yaml | 6 - .../style_05/b/mmlu_nutrition.yaml | 6 - .../style_05/b/mmlu_philosophy.yaml | 6 - .../style_05/b/mmlu_prehistory.yaml | 6 - .../b/mmlu_professional_accounting.yaml | 6 - .../style_05/b/mmlu_professional_law.yaml | 6 - .../b/mmlu_professional_medicine.yaml | 6 - .../b/mmlu_professional_psychology.yaml | 6 - .../style_05/b/mmlu_public_relations.yaml | 6 - .../style_05/b/mmlu_security_studies.yaml | 6 - .../style_05/b/mmlu_sociology.yaml | 6 - .../style_05/b/mmlu_us_foreign_policy.yaml | 6 - .../style_05/b/mmlu_virology.yaml | 6 - .../style_05/b/mmlu_world_religions.yaml | 6 - .../output_variation/style_05/c/_mmlu.yaml | 6 - .../style_05/c/_template_yaml | 11 -- .../style_05/c/mmlu_abstract_algebra.yaml | 6 - .../style_05/c/mmlu_anatomy.yaml | 6 - .../style_05/c/mmlu_astronomy.yaml | 6 - .../style_05/c/mmlu_business_ethics.yaml | 6 - .../style_05/c/mmlu_clinical_knowledge.yaml | 6 - .../style_05/c/mmlu_college_biology.yaml | 6 - .../style_05/c/mmlu_college_chemistry.yaml | 6 - .../c/mmlu_college_computer_science.yaml | 6 - .../style_05/c/mmlu_college_mathematics.yaml | 6 - .../style_05/c/mmlu_college_medicine.yaml | 6 - .../style_05/c/mmlu_college_physics.yaml | 6 - .../style_05/c/mmlu_computer_security.yaml | 6 - .../style_05/c/mmlu_conceptual_physics.yaml | 6 - .../style_05/c/mmlu_econometrics.yaml | 6 - .../c/mmlu_electrical_engineering.yaml | 6 - .../c/mmlu_elementary_mathematics.yaml | 6 - .../style_05/c/mmlu_formal_logic.yaml | 6 - .../style_05/c/mmlu_global_facts.yaml | 6 - .../style_05/c/mmlu_high_school_biology.yaml | 6 - .../c/mmlu_high_school_chemistry.yaml | 6 - .../c/mmlu_high_school_computer_science.yaml | 6 - .../c/mmlu_high_school_european_history.yaml | 6 - .../c/mmlu_high_school_geography.yaml | 6 - ...u_high_school_government_and_politics.yaml | 6 - .../c/mmlu_high_school_macroeconomics.yaml | 6 - .../c/mmlu_high_school_mathematics.yaml | 6 - .../c/mmlu_high_school_microeconomics.yaml | 6 - .../style_05/c/mmlu_high_school_physics.yaml | 6 - .../c/mmlu_high_school_psychology.yaml | 6 - .../c/mmlu_high_school_statistics.yaml | 6 - .../c/mmlu_high_school_us_history.yaml | 6 - .../c/mmlu_high_school_world_history.yaml | 6 - .../style_05/c/mmlu_human_aging.yaml | 6 - .../style_05/c/mmlu_human_sexuality.yaml | 6 - .../style_05/c/mmlu_international_law.yaml | 6 - .../style_05/c/mmlu_jurisprudence.yaml | 6 - .../style_05/c/mmlu_logical_fallacies.yaml | 6 - .../style_05/c/mmlu_machine_learning.yaml | 6 - .../style_05/c/mmlu_management.yaml | 6 - .../style_05/c/mmlu_marketing.yaml | 6 - .../style_05/c/mmlu_medical_genetics.yaml | 6 - .../style_05/c/mmlu_miscellaneous.yaml | 6 - .../style_05/c/mmlu_moral_disputes.yaml | 6 - .../style_05/c/mmlu_moral_scenarios.yaml | 6 - .../style_05/c/mmlu_nutrition.yaml | 6 - .../style_05/c/mmlu_philosophy.yaml | 6 - .../style_05/c/mmlu_prehistory.yaml | 6 - .../c/mmlu_professional_accounting.yaml | 6 - .../style_05/c/mmlu_professional_law.yaml | 6 - .../c/mmlu_professional_medicine.yaml | 6 - .../c/mmlu_professional_psychology.yaml | 6 - .../style_05/c/mmlu_public_relations.yaml | 6 - .../style_05/c/mmlu_security_studies.yaml | 6 - .../style_05/c/mmlu_sociology.yaml | 6 - .../style_05/c/mmlu_us_foreign_policy.yaml | 6 - .../style_05/c/mmlu_virology.yaml | 6 - .../style_05/c/mmlu_world_religions.yaml | 6 - .../style_05_generative/a/_mmlu.yaml | 6 - .../style_05_generative/a/_template_yaml | 18 --- .../a/mmlu_abstract_algebra.yaml | 6 - .../style_05_generative/a/mmlu_anatomy.yaml | 6 - .../style_05_generative/a/mmlu_astronomy.yaml | 6 - .../a/mmlu_business_ethics.yaml | 6 - .../a/mmlu_clinical_knowledge.yaml | 6 - .../a/mmlu_college_biology.yaml | 6 - .../a/mmlu_college_chemistry.yaml | 6 - .../a/mmlu_college_computer_science.yaml | 6 - .../a/mmlu_college_mathematics.yaml | 6 - .../a/mmlu_college_medicine.yaml | 6 - .../a/mmlu_college_physics.yaml | 6 - .../a/mmlu_computer_security.yaml | 6 - .../a/mmlu_conceptual_physics.yaml | 6 - .../a/mmlu_econometrics.yaml | 6 - .../a/mmlu_electrical_engineering.yaml | 6 - .../a/mmlu_elementary_mathematics.yaml | 6 - .../a/mmlu_formal_logic.yaml | 6 - .../a/mmlu_global_facts.yaml | 6 - .../a/mmlu_high_school_biology.yaml | 6 - .../a/mmlu_high_school_chemistry.yaml | 6 - .../a/mmlu_high_school_computer_science.yaml | 6 - .../a/mmlu_high_school_european_history.yaml | 6 - .../a/mmlu_high_school_geography.yaml | 6 - ...u_high_school_government_and_politics.yaml | 6 - .../a/mmlu_high_school_macroeconomics.yaml | 6 - .../a/mmlu_high_school_mathematics.yaml | 6 - .../a/mmlu_high_school_microeconomics.yaml | 6 - .../a/mmlu_high_school_physics.yaml | 6 - .../a/mmlu_high_school_psychology.yaml | 6 - .../a/mmlu_high_school_statistics.yaml | 6 - .../a/mmlu_high_school_us_history.yaml | 6 - .../a/mmlu_high_school_world_history.yaml | 6 - .../a/mmlu_human_aging.yaml | 6 - .../a/mmlu_human_sexuality.yaml | 6 - .../a/mmlu_international_law.yaml | 6 - .../a/mmlu_jurisprudence.yaml | 6 - .../a/mmlu_logical_fallacies.yaml | 6 - .../a/mmlu_machine_learning.yaml | 6 - .../a/mmlu_management.yaml | 6 - .../style_05_generative/a/mmlu_marketing.yaml | 6 - .../a/mmlu_medical_genetics.yaml | 6 - .../a/mmlu_miscellaneous.yaml | 6 - .../a/mmlu_moral_disputes.yaml | 6 - .../a/mmlu_moral_scenarios.yaml | 6 - .../style_05_generative/a/mmlu_nutrition.yaml | 6 - .../a/mmlu_philosophy.yaml | 6 - .../a/mmlu_prehistory.yaml | 6 - .../a/mmlu_professional_accounting.yaml | 6 - .../a/mmlu_professional_law.yaml | 6 - .../a/mmlu_professional_medicine.yaml | 6 - .../a/mmlu_professional_psychology.yaml | 6 - .../a/mmlu_public_relations.yaml | 6 - .../a/mmlu_security_studies.yaml | 6 - .../style_05_generative/a/mmlu_sociology.yaml | 6 - .../a/mmlu_us_foreign_policy.yaml | 6 - .../style_05_generative/a/mmlu_virology.yaml | 6 - .../a/mmlu_world_religions.yaml | 6 - .../style_06/_mmlu_ov_06.yaml | 5 - .../output_variation/style_06/a/_mmlu.yaml | 6 - .../style_06/a/_template_yaml | 11 -- .../style_06/a/mmlu_abstract_algebra.yaml | 6 - .../style_06/a/mmlu_anatomy.yaml | 6 - .../style_06/a/mmlu_astronomy.yaml | 6 - .../style_06/a/mmlu_business_ethics.yaml | 6 - .../style_06/a/mmlu_clinical_knowledge.yaml | 6 - .../style_06/a/mmlu_college_biology.yaml | 6 - .../style_06/a/mmlu_college_chemistry.yaml | 6 - .../a/mmlu_college_computer_science.yaml | 6 - .../style_06/a/mmlu_college_mathematics.yaml | 6 - .../style_06/a/mmlu_college_medicine.yaml | 6 - .../style_06/a/mmlu_college_physics.yaml | 6 - .../style_06/a/mmlu_computer_security.yaml | 6 - .../style_06/a/mmlu_conceptual_physics.yaml | 6 - .../style_06/a/mmlu_econometrics.yaml | 6 - .../a/mmlu_electrical_engineering.yaml | 6 - .../a/mmlu_elementary_mathematics.yaml | 6 - .../style_06/a/mmlu_formal_logic.yaml | 6 - .../style_06/a/mmlu_global_facts.yaml | 6 - .../style_06/a/mmlu_high_school_biology.yaml | 6 - .../a/mmlu_high_school_chemistry.yaml | 6 - .../a/mmlu_high_school_computer_science.yaml | 6 - .../a/mmlu_high_school_european_history.yaml | 6 - .../a/mmlu_high_school_geography.yaml | 6 - ...u_high_school_government_and_politics.yaml | 6 - .../a/mmlu_high_school_macroeconomics.yaml | 6 - .../a/mmlu_high_school_mathematics.yaml | 6 - .../a/mmlu_high_school_microeconomics.yaml | 6 - .../style_06/a/mmlu_high_school_physics.yaml | 6 - .../a/mmlu_high_school_psychology.yaml | 6 - .../a/mmlu_high_school_statistics.yaml | 6 - .../a/mmlu_high_school_us_history.yaml | 6 - .../a/mmlu_high_school_world_history.yaml | 6 - .../style_06/a/mmlu_human_aging.yaml | 6 - .../style_06/a/mmlu_human_sexuality.yaml | 6 - .../style_06/a/mmlu_international_law.yaml | 6 - .../style_06/a/mmlu_jurisprudence.yaml | 6 - .../style_06/a/mmlu_logical_fallacies.yaml | 6 - .../style_06/a/mmlu_machine_learning.yaml | 6 - .../style_06/a/mmlu_management.yaml | 6 - .../style_06/a/mmlu_marketing.yaml | 6 - .../style_06/a/mmlu_medical_genetics.yaml | 6 - .../style_06/a/mmlu_miscellaneous.yaml | 6 - .../style_06/a/mmlu_moral_disputes.yaml | 6 - .../style_06/a/mmlu_moral_scenarios.yaml | 6 - .../style_06/a/mmlu_nutrition.yaml | 6 - .../style_06/a/mmlu_philosophy.yaml | 6 - .../style_06/a/mmlu_prehistory.yaml | 6 - .../a/mmlu_professional_accounting.yaml | 6 - .../style_06/a/mmlu_professional_law.yaml | 6 - .../a/mmlu_professional_medicine.yaml | 6 - .../a/mmlu_professional_psychology.yaml | 6 - .../style_06/a/mmlu_public_relations.yaml | 6 - .../style_06/a/mmlu_security_studies.yaml | 6 - .../style_06/a/mmlu_sociology.yaml | 6 - .../style_06/a/mmlu_us_foreign_policy.yaml | 6 - .../style_06/a/mmlu_virology.yaml | 6 - .../style_06/a/mmlu_world_religions.yaml | 6 - .../output_variation/style_06/b/_mmlu.yaml | 6 - .../style_06/b/_template_yaml | 11 -- .../style_06/b/mmlu_abstract_algebra.yaml | 6 - .../style_06/b/mmlu_anatomy.yaml | 6 - .../style_06/b/mmlu_astronomy.yaml | 6 - .../style_06/b/mmlu_business_ethics.yaml | 6 - .../style_06/b/mmlu_clinical_knowledge.yaml | 6 - .../style_06/b/mmlu_college_biology.yaml | 6 - .../style_06/b/mmlu_college_chemistry.yaml | 6 - .../b/mmlu_college_computer_science.yaml | 6 - .../style_06/b/mmlu_college_mathematics.yaml | 6 - .../style_06/b/mmlu_college_medicine.yaml | 6 - .../style_06/b/mmlu_college_physics.yaml | 6 - .../style_06/b/mmlu_computer_security.yaml | 6 - .../style_06/b/mmlu_conceptual_physics.yaml | 6 - .../style_06/b/mmlu_econometrics.yaml | 6 - .../b/mmlu_electrical_engineering.yaml | 6 - .../b/mmlu_elementary_mathematics.yaml | 6 - .../style_06/b/mmlu_formal_logic.yaml | 6 - .../style_06/b/mmlu_global_facts.yaml | 6 - .../style_06/b/mmlu_high_school_biology.yaml | 6 - .../b/mmlu_high_school_chemistry.yaml | 6 - .../b/mmlu_high_school_computer_science.yaml | 6 - .../b/mmlu_high_school_european_history.yaml | 6 - .../b/mmlu_high_school_geography.yaml | 6 - ...u_high_school_government_and_politics.yaml | 6 - .../b/mmlu_high_school_macroeconomics.yaml | 6 - .../b/mmlu_high_school_mathematics.yaml | 6 - .../b/mmlu_high_school_microeconomics.yaml | 6 - .../style_06/b/mmlu_high_school_physics.yaml | 6 - .../b/mmlu_high_school_psychology.yaml | 6 - .../b/mmlu_high_school_statistics.yaml | 6 - .../b/mmlu_high_school_us_history.yaml | 6 - .../b/mmlu_high_school_world_history.yaml | 6 - .../style_06/b/mmlu_human_aging.yaml | 6 - .../style_06/b/mmlu_human_sexuality.yaml | 6 - .../style_06/b/mmlu_international_law.yaml | 6 - .../style_06/b/mmlu_jurisprudence.yaml | 6 - .../style_06/b/mmlu_logical_fallacies.yaml | 6 - .../style_06/b/mmlu_machine_learning.yaml | 6 - .../style_06/b/mmlu_management.yaml | 6 - .../style_06/b/mmlu_marketing.yaml | 6 - .../style_06/b/mmlu_medical_genetics.yaml | 6 - .../style_06/b/mmlu_miscellaneous.yaml | 6 - .../style_06/b/mmlu_moral_disputes.yaml | 6 - .../style_06/b/mmlu_moral_scenarios.yaml | 6 - .../style_06/b/mmlu_nutrition.yaml | 6 - .../style_06/b/mmlu_philosophy.yaml | 6 - .../style_06/b/mmlu_prehistory.yaml | 6 - .../b/mmlu_professional_accounting.yaml | 6 - .../style_06/b/mmlu_professional_law.yaml | 6 - .../b/mmlu_professional_medicine.yaml | 6 - .../b/mmlu_professional_psychology.yaml | 6 - .../style_06/b/mmlu_public_relations.yaml | 6 - .../style_06/b/mmlu_security_studies.yaml | 6 - .../style_06/b/mmlu_sociology.yaml | 6 - .../style_06/b/mmlu_us_foreign_policy.yaml | 6 - .../style_06/b/mmlu_virology.yaml | 6 - .../style_06/b/mmlu_world_religions.yaml | 6 - .../output_variation/style_06/c/_mmlu.yaml | 6 - .../style_06/c/_template_yaml | 11 -- .../style_06/c/mmlu_abstract_algebra.yaml | 6 - .../style_06/c/mmlu_anatomy.yaml | 6 - .../style_06/c/mmlu_astronomy.yaml | 6 - .../style_06/c/mmlu_business_ethics.yaml | 6 - .../style_06/c/mmlu_clinical_knowledge.yaml | 6 - .../style_06/c/mmlu_college_biology.yaml | 6 - .../style_06/c/mmlu_college_chemistry.yaml | 6 - .../c/mmlu_college_computer_science.yaml | 6 - .../style_06/c/mmlu_college_mathematics.yaml | 6 - .../style_06/c/mmlu_college_medicine.yaml | 6 - .../style_06/c/mmlu_college_physics.yaml | 6 - .../style_06/c/mmlu_computer_security.yaml | 6 - .../style_06/c/mmlu_conceptual_physics.yaml | 6 - .../style_06/c/mmlu_econometrics.yaml | 6 - .../c/mmlu_electrical_engineering.yaml | 6 - .../c/mmlu_elementary_mathematics.yaml | 6 - .../style_06/c/mmlu_formal_logic.yaml | 6 - .../style_06/c/mmlu_global_facts.yaml | 6 - .../style_06/c/mmlu_high_school_biology.yaml | 6 - .../c/mmlu_high_school_chemistry.yaml | 6 - .../c/mmlu_high_school_computer_science.yaml | 6 - .../c/mmlu_high_school_european_history.yaml | 6 - .../c/mmlu_high_school_geography.yaml | 6 - ...u_high_school_government_and_politics.yaml | 6 - .../c/mmlu_high_school_macroeconomics.yaml | 6 - .../c/mmlu_high_school_mathematics.yaml | 6 - .../c/mmlu_high_school_microeconomics.yaml | 6 - .../style_06/c/mmlu_high_school_physics.yaml | 6 - .../c/mmlu_high_school_psychology.yaml | 6 - .../c/mmlu_high_school_statistics.yaml | 6 - .../c/mmlu_high_school_us_history.yaml | 6 - .../c/mmlu_high_school_world_history.yaml | 6 - .../style_06/c/mmlu_human_aging.yaml | 6 - .../style_06/c/mmlu_human_sexuality.yaml | 6 - .../style_06/c/mmlu_international_law.yaml | 6 - .../style_06/c/mmlu_jurisprudence.yaml | 6 - .../style_06/c/mmlu_logical_fallacies.yaml | 6 - .../style_06/c/mmlu_machine_learning.yaml | 6 - .../style_06/c/mmlu_management.yaml | 6 - .../style_06/c/mmlu_marketing.yaml | 6 - .../style_06/c/mmlu_medical_genetics.yaml | 6 - .../style_06/c/mmlu_miscellaneous.yaml | 6 - .../style_06/c/mmlu_moral_disputes.yaml | 6 - .../style_06/c/mmlu_moral_scenarios.yaml | 6 - .../style_06/c/mmlu_nutrition.yaml | 6 - .../style_06/c/mmlu_philosophy.yaml | 6 - .../style_06/c/mmlu_prehistory.yaml | 6 - .../c/mmlu_professional_accounting.yaml | 6 - .../style_06/c/mmlu_professional_law.yaml | 6 - .../c/mmlu_professional_medicine.yaml | 6 - .../c/mmlu_professional_psychology.yaml | 6 - .../style_06/c/mmlu_public_relations.yaml | 6 - .../style_06/c/mmlu_security_studies.yaml | 6 - .../style_06/c/mmlu_sociology.yaml | 6 - .../style_06/c/mmlu_us_foreign_policy.yaml | 6 - .../style_06/c/mmlu_virology.yaml | 6 - .../style_06/c/mmlu_world_religions.yaml | 6 - .../style_07/_mmlu_ov_07.yaml | 5 - .../output_variation/style_07/a/_mmlu.yaml | 6 - .../style_07/a/_template_yaml | 11 -- .../style_07/a/mmlu_abstract_algebra.yaml | 6 - .../style_07/a/mmlu_anatomy.yaml | 6 - .../style_07/a/mmlu_astronomy.yaml | 6 - .../style_07/a/mmlu_business_ethics.yaml | 6 - .../style_07/a/mmlu_clinical_knowledge.yaml | 6 - .../style_07/a/mmlu_college_biology.yaml | 6 - .../style_07/a/mmlu_college_chemistry.yaml | 6 - .../a/mmlu_college_computer_science.yaml | 6 - .../style_07/a/mmlu_college_mathematics.yaml | 6 - .../style_07/a/mmlu_college_medicine.yaml | 6 - .../style_07/a/mmlu_college_physics.yaml | 6 - .../style_07/a/mmlu_computer_security.yaml | 6 - .../style_07/a/mmlu_conceptual_physics.yaml | 6 - .../style_07/a/mmlu_econometrics.yaml | 6 - .../a/mmlu_electrical_engineering.yaml | 6 - .../a/mmlu_elementary_mathematics.yaml | 6 - .../style_07/a/mmlu_formal_logic.yaml | 6 - .../style_07/a/mmlu_global_facts.yaml | 6 - .../style_07/a/mmlu_high_school_biology.yaml | 6 - .../a/mmlu_high_school_chemistry.yaml | 6 - .../a/mmlu_high_school_computer_science.yaml | 6 - .../a/mmlu_high_school_european_history.yaml | 6 - .../a/mmlu_high_school_geography.yaml | 6 - ...u_high_school_government_and_politics.yaml | 6 - .../a/mmlu_high_school_macroeconomics.yaml | 6 - .../a/mmlu_high_school_mathematics.yaml | 6 - .../a/mmlu_high_school_microeconomics.yaml | 6 - .../style_07/a/mmlu_high_school_physics.yaml | 6 - .../a/mmlu_high_school_psychology.yaml | 6 - .../a/mmlu_high_school_statistics.yaml | 6 - .../a/mmlu_high_school_us_history.yaml | 6 - .../a/mmlu_high_school_world_history.yaml | 6 - .../style_07/a/mmlu_human_aging.yaml | 6 - .../style_07/a/mmlu_human_sexuality.yaml | 6 - .../style_07/a/mmlu_international_law.yaml | 6 - .../style_07/a/mmlu_jurisprudence.yaml | 6 - .../style_07/a/mmlu_logical_fallacies.yaml | 6 - .../style_07/a/mmlu_machine_learning.yaml | 6 - .../style_07/a/mmlu_management.yaml | 6 - .../style_07/a/mmlu_marketing.yaml | 6 - .../style_07/a/mmlu_medical_genetics.yaml | 6 - .../style_07/a/mmlu_miscellaneous.yaml | 6 - .../style_07/a/mmlu_moral_disputes.yaml | 6 - .../style_07/a/mmlu_moral_scenarios.yaml | 6 - .../style_07/a/mmlu_nutrition.yaml | 6 - .../style_07/a/mmlu_philosophy.yaml | 6 - .../style_07/a/mmlu_prehistory.yaml | 6 - .../a/mmlu_professional_accounting.yaml | 6 - .../style_07/a/mmlu_professional_law.yaml | 6 - .../a/mmlu_professional_medicine.yaml | 6 - .../a/mmlu_professional_psychology.yaml | 6 - .../style_07/a/mmlu_public_relations.yaml | 6 - .../style_07/a/mmlu_security_studies.yaml | 6 - .../style_07/a/mmlu_sociology.yaml | 6 - .../style_07/a/mmlu_us_foreign_policy.yaml | 6 - .../style_07/a/mmlu_virology.yaml | 6 - .../style_07/a/mmlu_world_religions.yaml | 6 - .../output_variation/style_07/b/_mmlu.yaml | 6 - .../style_07/b/_template_yaml | 11 -- .../style_07/b/mmlu_abstract_algebra.yaml | 6 - .../style_07/b/mmlu_anatomy.yaml | 6 - .../style_07/b/mmlu_astronomy.yaml | 6 - .../style_07/b/mmlu_business_ethics.yaml | 6 - .../style_07/b/mmlu_clinical_knowledge.yaml | 6 - .../style_07/b/mmlu_college_biology.yaml | 6 - .../style_07/b/mmlu_college_chemistry.yaml | 6 - .../b/mmlu_college_computer_science.yaml | 6 - .../style_07/b/mmlu_college_mathematics.yaml | 6 - .../style_07/b/mmlu_college_medicine.yaml | 6 - .../style_07/b/mmlu_college_physics.yaml | 6 - .../style_07/b/mmlu_computer_security.yaml | 6 - .../style_07/b/mmlu_conceptual_physics.yaml | 6 - .../style_07/b/mmlu_econometrics.yaml | 6 - .../b/mmlu_electrical_engineering.yaml | 6 - .../b/mmlu_elementary_mathematics.yaml | 6 - .../style_07/b/mmlu_formal_logic.yaml | 6 - .../style_07/b/mmlu_global_facts.yaml | 6 - .../style_07/b/mmlu_high_school_biology.yaml | 6 - .../b/mmlu_high_school_chemistry.yaml | 6 - .../b/mmlu_high_school_computer_science.yaml | 6 - .../b/mmlu_high_school_european_history.yaml | 6 - .../b/mmlu_high_school_geography.yaml | 6 - ...u_high_school_government_and_politics.yaml | 6 - .../b/mmlu_high_school_macroeconomics.yaml | 6 - .../b/mmlu_high_school_mathematics.yaml | 6 - .../b/mmlu_high_school_microeconomics.yaml | 6 - .../style_07/b/mmlu_high_school_physics.yaml | 6 - .../b/mmlu_high_school_psychology.yaml | 6 - .../b/mmlu_high_school_statistics.yaml | 6 - .../b/mmlu_high_school_us_history.yaml | 6 - .../b/mmlu_high_school_world_history.yaml | 6 - .../style_07/b/mmlu_human_aging.yaml | 6 - .../style_07/b/mmlu_human_sexuality.yaml | 6 - .../style_07/b/mmlu_international_law.yaml | 6 - .../style_07/b/mmlu_jurisprudence.yaml | 6 - .../style_07/b/mmlu_logical_fallacies.yaml | 6 - .../style_07/b/mmlu_machine_learning.yaml | 6 - .../style_07/b/mmlu_management.yaml | 6 - .../style_07/b/mmlu_marketing.yaml | 6 - .../style_07/b/mmlu_medical_genetics.yaml | 6 - .../style_07/b/mmlu_miscellaneous.yaml | 6 - .../style_07/b/mmlu_moral_disputes.yaml | 6 - .../style_07/b/mmlu_moral_scenarios.yaml | 6 - .../style_07/b/mmlu_nutrition.yaml | 6 - .../style_07/b/mmlu_philosophy.yaml | 6 - .../style_07/b/mmlu_prehistory.yaml | 6 - .../b/mmlu_professional_accounting.yaml | 6 - .../style_07/b/mmlu_professional_law.yaml | 6 - .../b/mmlu_professional_medicine.yaml | 6 - .../b/mmlu_professional_psychology.yaml | 6 - .../style_07/b/mmlu_public_relations.yaml | 6 - .../style_07/b/mmlu_security_studies.yaml | 6 - .../style_07/b/mmlu_sociology.yaml | 6 - .../style_07/b/mmlu_us_foreign_policy.yaml | 6 - .../style_07/b/mmlu_virology.yaml | 6 - .../style_07/b/mmlu_world_religions.yaml | 6 - .../output_variation/style_07/c/_mmlu.yaml | 6 - .../style_07/c/_template_yaml | 11 -- .../style_07/c/mmlu_abstract_algebra.yaml | 6 - .../style_07/c/mmlu_anatomy.yaml | 6 - .../style_07/c/mmlu_astronomy.yaml | 6 - .../style_07/c/mmlu_business_ethics.yaml | 6 - .../style_07/c/mmlu_clinical_knowledge.yaml | 6 - .../style_07/c/mmlu_college_biology.yaml | 6 - .../style_07/c/mmlu_college_chemistry.yaml | 6 - .../c/mmlu_college_computer_science.yaml | 6 - .../style_07/c/mmlu_college_mathematics.yaml | 6 - .../style_07/c/mmlu_college_medicine.yaml | 6 - .../style_07/c/mmlu_college_physics.yaml | 6 - .../style_07/c/mmlu_computer_security.yaml | 6 - .../style_07/c/mmlu_conceptual_physics.yaml | 6 - .../style_07/c/mmlu_econometrics.yaml | 6 - .../c/mmlu_electrical_engineering.yaml | 6 - .../c/mmlu_elementary_mathematics.yaml | 6 - .../style_07/c/mmlu_formal_logic.yaml | 6 - .../style_07/c/mmlu_global_facts.yaml | 6 - .../style_07/c/mmlu_high_school_biology.yaml | 6 - .../c/mmlu_high_school_chemistry.yaml | 6 - .../c/mmlu_high_school_computer_science.yaml | 6 - .../c/mmlu_high_school_european_history.yaml | 6 - .../c/mmlu_high_school_geography.yaml | 6 - ...u_high_school_government_and_politics.yaml | 6 - .../c/mmlu_high_school_macroeconomics.yaml | 6 - .../c/mmlu_high_school_mathematics.yaml | 6 - .../c/mmlu_high_school_microeconomics.yaml | 6 - .../style_07/c/mmlu_high_school_physics.yaml | 6 - .../c/mmlu_high_school_psychology.yaml | 6 - .../c/mmlu_high_school_statistics.yaml | 6 - .../c/mmlu_high_school_us_history.yaml | 6 - .../c/mmlu_high_school_world_history.yaml | 6 - .../style_07/c/mmlu_human_aging.yaml | 6 - .../style_07/c/mmlu_human_sexuality.yaml | 6 - .../style_07/c/mmlu_international_law.yaml | 6 - .../style_07/c/mmlu_jurisprudence.yaml | 6 - .../style_07/c/mmlu_logical_fallacies.yaml | 6 - .../style_07/c/mmlu_machine_learning.yaml | 6 - .../style_07/c/mmlu_management.yaml | 6 - .../style_07/c/mmlu_marketing.yaml | 6 - .../style_07/c/mmlu_medical_genetics.yaml | 6 - .../style_07/c/mmlu_miscellaneous.yaml | 6 - .../style_07/c/mmlu_moral_disputes.yaml | 6 - .../style_07/c/mmlu_moral_scenarios.yaml | 6 - .../style_07/c/mmlu_nutrition.yaml | 6 - .../style_07/c/mmlu_philosophy.yaml | 6 - .../style_07/c/mmlu_prehistory.yaml | 6 - .../c/mmlu_professional_accounting.yaml | 6 - .../style_07/c/mmlu_professional_law.yaml | 6 - .../c/mmlu_professional_medicine.yaml | 6 - .../c/mmlu_professional_psychology.yaml | 6 - .../style_07/c/mmlu_public_relations.yaml | 6 - .../style_07/c/mmlu_security_studies.yaml | 6 - .../style_07/c/mmlu_sociology.yaml | 6 - .../style_07/c/mmlu_us_foreign_policy.yaml | 6 - .../style_07/c/mmlu_virology.yaml | 6 - .../style_07/c/mmlu_world_religions.yaml | 6 - .../style_08/_mmlu_ov_08.yaml | 5 - .../output_variation/style_08/a/_mmlu.yaml | 6 - .../style_08/a/_template_yaml | 11 -- .../style_08/a/mmlu_abstract_algebra.yaml | 6 - .../style_08/a/mmlu_anatomy.yaml | 6 - .../style_08/a/mmlu_astronomy.yaml | 6 - .../style_08/a/mmlu_business_ethics.yaml | 6 - .../style_08/a/mmlu_clinical_knowledge.yaml | 6 - .../style_08/a/mmlu_college_biology.yaml | 6 - .../style_08/a/mmlu_college_chemistry.yaml | 6 - .../a/mmlu_college_computer_science.yaml | 6 - .../style_08/a/mmlu_college_mathematics.yaml | 6 - .../style_08/a/mmlu_college_medicine.yaml | 6 - .../style_08/a/mmlu_college_physics.yaml | 6 - .../style_08/a/mmlu_computer_security.yaml | 6 - .../style_08/a/mmlu_conceptual_physics.yaml | 6 - .../style_08/a/mmlu_econometrics.yaml | 6 - .../a/mmlu_electrical_engineering.yaml | 6 - .../a/mmlu_elementary_mathematics.yaml | 6 - .../style_08/a/mmlu_formal_logic.yaml | 6 - .../style_08/a/mmlu_global_facts.yaml | 6 - .../style_08/a/mmlu_high_school_biology.yaml | 6 - .../a/mmlu_high_school_chemistry.yaml | 6 - .../a/mmlu_high_school_computer_science.yaml | 6 - .../a/mmlu_high_school_european_history.yaml | 6 - .../a/mmlu_high_school_geography.yaml | 6 - ...u_high_school_government_and_politics.yaml | 6 - .../a/mmlu_high_school_macroeconomics.yaml | 6 - .../a/mmlu_high_school_mathematics.yaml | 6 - .../a/mmlu_high_school_microeconomics.yaml | 6 - .../style_08/a/mmlu_high_school_physics.yaml | 6 - .../a/mmlu_high_school_psychology.yaml | 6 - .../a/mmlu_high_school_statistics.yaml | 6 - .../a/mmlu_high_school_us_history.yaml | 6 - .../a/mmlu_high_school_world_history.yaml | 6 - .../style_08/a/mmlu_human_aging.yaml | 6 - .../style_08/a/mmlu_human_sexuality.yaml | 6 - .../style_08/a/mmlu_international_law.yaml | 6 - .../style_08/a/mmlu_jurisprudence.yaml | 6 - .../style_08/a/mmlu_logical_fallacies.yaml | 6 - .../style_08/a/mmlu_machine_learning.yaml | 6 - .../style_08/a/mmlu_management.yaml | 6 - .../style_08/a/mmlu_marketing.yaml | 6 - .../style_08/a/mmlu_medical_genetics.yaml | 6 - .../style_08/a/mmlu_miscellaneous.yaml | 6 - .../style_08/a/mmlu_moral_disputes.yaml | 6 - .../style_08/a/mmlu_moral_scenarios.yaml | 6 - .../style_08/a/mmlu_nutrition.yaml | 6 - .../style_08/a/mmlu_philosophy.yaml | 6 - .../style_08/a/mmlu_prehistory.yaml | 6 - .../a/mmlu_professional_accounting.yaml | 6 - .../style_08/a/mmlu_professional_law.yaml | 6 - .../a/mmlu_professional_medicine.yaml | 6 - .../a/mmlu_professional_psychology.yaml | 6 - .../style_08/a/mmlu_public_relations.yaml | 6 - .../style_08/a/mmlu_security_studies.yaml | 6 - .../style_08/a/mmlu_sociology.yaml | 6 - .../style_08/a/mmlu_us_foreign_policy.yaml | 6 - .../style_08/a/mmlu_virology.yaml | 6 - .../style_08/a/mmlu_world_religions.yaml | 6 - .../output_variation/style_08/b/_mmlu.yaml | 6 - .../style_08/b/_template_yaml | 11 -- .../style_08/b/mmlu_abstract_algebra.yaml | 6 - .../style_08/b/mmlu_anatomy.yaml | 6 - .../style_08/b/mmlu_astronomy.yaml | 6 - .../style_08/b/mmlu_business_ethics.yaml | 6 - .../style_08/b/mmlu_clinical_knowledge.yaml | 6 - .../style_08/b/mmlu_college_biology.yaml | 6 - .../style_08/b/mmlu_college_chemistry.yaml | 6 - .../b/mmlu_college_computer_science.yaml | 6 - .../style_08/b/mmlu_college_mathematics.yaml | 6 - .../style_08/b/mmlu_college_medicine.yaml | 6 - .../style_08/b/mmlu_college_physics.yaml | 6 - .../style_08/b/mmlu_computer_security.yaml | 6 - .../style_08/b/mmlu_conceptual_physics.yaml | 6 - .../style_08/b/mmlu_econometrics.yaml | 6 - .../b/mmlu_electrical_engineering.yaml | 6 - .../b/mmlu_elementary_mathematics.yaml | 6 - .../style_08/b/mmlu_formal_logic.yaml | 6 - .../style_08/b/mmlu_global_facts.yaml | 6 - .../style_08/b/mmlu_high_school_biology.yaml | 6 - .../b/mmlu_high_school_chemistry.yaml | 6 - .../b/mmlu_high_school_computer_science.yaml | 6 - .../b/mmlu_high_school_european_history.yaml | 6 - .../b/mmlu_high_school_geography.yaml | 6 - ...u_high_school_government_and_politics.yaml | 6 - .../b/mmlu_high_school_macroeconomics.yaml | 6 - .../b/mmlu_high_school_mathematics.yaml | 6 - .../b/mmlu_high_school_microeconomics.yaml | 6 - .../style_08/b/mmlu_high_school_physics.yaml | 6 - .../b/mmlu_high_school_psychology.yaml | 6 - .../b/mmlu_high_school_statistics.yaml | 6 - .../b/mmlu_high_school_us_history.yaml | 6 - .../b/mmlu_high_school_world_history.yaml | 6 - .../style_08/b/mmlu_human_aging.yaml | 6 - .../style_08/b/mmlu_human_sexuality.yaml | 6 - .../style_08/b/mmlu_international_law.yaml | 6 - .../style_08/b/mmlu_jurisprudence.yaml | 6 - .../style_08/b/mmlu_logical_fallacies.yaml | 6 - .../style_08/b/mmlu_machine_learning.yaml | 6 - .../style_08/b/mmlu_management.yaml | 6 - .../style_08/b/mmlu_marketing.yaml | 6 - .../style_08/b/mmlu_medical_genetics.yaml | 6 - .../style_08/b/mmlu_miscellaneous.yaml | 6 - .../style_08/b/mmlu_moral_disputes.yaml | 6 - .../style_08/b/mmlu_moral_scenarios.yaml | 6 - .../style_08/b/mmlu_nutrition.yaml | 6 - .../style_08/b/mmlu_philosophy.yaml | 6 - .../style_08/b/mmlu_prehistory.yaml | 6 - .../b/mmlu_professional_accounting.yaml | 6 - .../style_08/b/mmlu_professional_law.yaml | 6 - .../b/mmlu_professional_medicine.yaml | 6 - .../b/mmlu_professional_psychology.yaml | 6 - .../style_08/b/mmlu_public_relations.yaml | 6 - .../style_08/b/mmlu_security_studies.yaml | 6 - .../style_08/b/mmlu_sociology.yaml | 6 - .../style_08/b/mmlu_us_foreign_policy.yaml | 6 - .../style_08/b/mmlu_virology.yaml | 6 - .../style_08/b/mmlu_world_religions.yaml | 6 - .../output_variation/style_08/c/_mmlu.yaml | 6 - .../style_08/c/_template_yaml | 11 -- .../style_08/c/mmlu_abstract_algebra.yaml | 6 - .../style_08/c/mmlu_anatomy.yaml | 6 - .../style_08/c/mmlu_astronomy.yaml | 6 - .../style_08/c/mmlu_business_ethics.yaml | 6 - .../style_08/c/mmlu_clinical_knowledge.yaml | 6 - .../style_08/c/mmlu_college_biology.yaml | 6 - .../style_08/c/mmlu_college_chemistry.yaml | 6 - .../c/mmlu_college_computer_science.yaml | 6 - .../style_08/c/mmlu_college_mathematics.yaml | 6 - .../style_08/c/mmlu_college_medicine.yaml | 6 - .../style_08/c/mmlu_college_physics.yaml | 6 - .../style_08/c/mmlu_computer_security.yaml | 6 - .../style_08/c/mmlu_conceptual_physics.yaml | 6 - .../style_08/c/mmlu_econometrics.yaml | 6 - .../c/mmlu_electrical_engineering.yaml | 6 - .../c/mmlu_elementary_mathematics.yaml | 6 - .../style_08/c/mmlu_formal_logic.yaml | 6 - .../style_08/c/mmlu_global_facts.yaml | 6 - .../style_08/c/mmlu_high_school_biology.yaml | 6 - .../c/mmlu_high_school_chemistry.yaml | 6 - .../c/mmlu_high_school_computer_science.yaml | 6 - .../c/mmlu_high_school_european_history.yaml | 6 - .../c/mmlu_high_school_geography.yaml | 6 - ...u_high_school_government_and_politics.yaml | 6 - .../c/mmlu_high_school_macroeconomics.yaml | 6 - .../c/mmlu_high_school_mathematics.yaml | 6 - .../c/mmlu_high_school_microeconomics.yaml | 6 - .../style_08/c/mmlu_high_school_physics.yaml | 6 - .../c/mmlu_high_school_psychology.yaml | 6 - .../c/mmlu_high_school_statistics.yaml | 6 - .../c/mmlu_high_school_us_history.yaml | 6 - .../c/mmlu_high_school_world_history.yaml | 6 - .../style_08/c/mmlu_human_aging.yaml | 6 - .../style_08/c/mmlu_human_sexuality.yaml | 6 - .../style_08/c/mmlu_international_law.yaml | 6 - .../style_08/c/mmlu_jurisprudence.yaml | 6 - .../style_08/c/mmlu_logical_fallacies.yaml | 6 - .../style_08/c/mmlu_machine_learning.yaml | 6 - .../style_08/c/mmlu_management.yaml | 6 - .../style_08/c/mmlu_marketing.yaml | 6 - .../style_08/c/mmlu_medical_genetics.yaml | 6 - .../style_08/c/mmlu_miscellaneous.yaml | 6 - .../style_08/c/mmlu_moral_disputes.yaml | 6 - .../style_08/c/mmlu_moral_scenarios.yaml | 6 - .../style_08/c/mmlu_nutrition.yaml | 6 - .../style_08/c/mmlu_philosophy.yaml | 6 - .../style_08/c/mmlu_prehistory.yaml | 6 - .../c/mmlu_professional_accounting.yaml | 6 - .../style_08/c/mmlu_professional_law.yaml | 6 - .../c/mmlu_professional_medicine.yaml | 6 - .../c/mmlu_professional_psychology.yaml | 6 - .../style_08/c/mmlu_public_relations.yaml | 6 - .../style_08/c/mmlu_security_studies.yaml | 6 - .../style_08/c/mmlu_sociology.yaml | 6 - .../style_08/c/mmlu_us_foreign_policy.yaml | 6 - .../style_08/c/mmlu_virology.yaml | 6 - .../style_08/c/mmlu_world_religions.yaml | 6 - .../style_01/_default_template_yaml | 13 -- .../prompt_variation/style_01/_mmlu.yaml | 6 - .../style_01/mmlu_abstract_algebra.yaml | 6 - .../style_01/mmlu_anatomy.yaml | 6 - .../style_01/mmlu_astronomy.yaml | 6 - .../style_01/mmlu_business_ethics.yaml | 6 - .../style_01/mmlu_clinical_knowledge.yaml | 6 - .../style_01/mmlu_college_biology.yaml | 6 - .../style_01/mmlu_college_chemistry.yaml | 6 - .../mmlu_college_computer_science.yaml | 6 - .../style_01/mmlu_college_mathematics.yaml | 6 - .../style_01/mmlu_college_medicine.yaml | 6 - .../style_01/mmlu_college_physics.yaml | 6 - .../style_01/mmlu_computer_security.yaml | 6 - .../style_01/mmlu_conceptual_physics.yaml | 6 - .../style_01/mmlu_econometrics.yaml | 6 - .../style_01/mmlu_electrical_engineering.yaml | 6 - .../style_01/mmlu_elementary_mathematics.yaml | 6 - .../style_01/mmlu_formal_logic.yaml | 6 - .../style_01/mmlu_global_facts.yaml | 6 - .../style_01/mmlu_high_school_biology.yaml | 6 - .../style_01/mmlu_high_school_chemistry.yaml | 6 - .../mmlu_high_school_computer_science.yaml | 6 - .../mmlu_high_school_european_history.yaml | 6 - .../style_01/mmlu_high_school_geography.yaml | 6 - ...u_high_school_government_and_politics.yaml | 6 - .../mmlu_high_school_macroeconomics.yaml | 6 - .../mmlu_high_school_mathematics.yaml | 6 - .../mmlu_high_school_microeconomics.yaml | 6 - .../style_01/mmlu_high_school_physics.yaml | 6 - .../style_01/mmlu_high_school_psychology.yaml | 6 - .../style_01/mmlu_high_school_statistics.yaml | 6 - .../style_01/mmlu_high_school_us_history.yaml | 6 - .../mmlu_high_school_world_history.yaml | 6 - .../style_01/mmlu_human_aging.yaml | 6 - .../style_01/mmlu_human_sexuality.yaml | 6 - .../style_01/mmlu_international_law.yaml | 6 - .../style_01/mmlu_jurisprudence.yaml | 6 - .../style_01/mmlu_logical_fallacies.yaml | 6 - .../style_01/mmlu_machine_learning.yaml | 6 - .../style_01/mmlu_management.yaml | 6 - .../style_01/mmlu_marketing.yaml | 6 - .../style_01/mmlu_medical_genetics.yaml | 6 - .../style_01/mmlu_miscellaneous.yaml | 6 - .../style_01/mmlu_moral_disputes.yaml | 6 - .../style_01/mmlu_moral_scenarios.yaml | 6 - .../style_01/mmlu_nutrition.yaml | 6 - .../style_01/mmlu_philosophy.yaml | 6 - .../style_01/mmlu_prehistory.yaml | 6 - .../mmlu_professional_accounting.yaml | 6 - .../style_01/mmlu_professional_law.yaml | 6 - .../style_01/mmlu_professional_medicine.yaml | 6 - .../mmlu_professional_psychology.yaml | 6 - .../style_01/mmlu_public_relations.yaml | 6 - .../style_01/mmlu_security_studies.yaml | 6 - .../style_01/mmlu_sociology.yaml | 6 - .../style_01/mmlu_us_foreign_policy.yaml | 6 - .../style_01/mmlu_virology.yaml | 6 - .../style_01/mmlu_world_religions.yaml | 6 - .../style_02/_default_template_yaml | 13 -- .../prompt_variation/style_02/_mmlu.yaml | 6 - .../style_02/mmlu_abstract_algebra.yaml | 6 - .../style_02/mmlu_anatomy.yaml | 6 - .../style_02/mmlu_astronomy.yaml | 6 - .../style_02/mmlu_business_ethics.yaml | 6 - .../style_02/mmlu_clinical_knowledge.yaml | 6 - .../style_02/mmlu_college_biology.yaml | 6 - .../style_02/mmlu_college_chemistry.yaml | 6 - .../mmlu_college_computer_science.yaml | 6 - .../style_02/mmlu_college_mathematics.yaml | 6 - .../style_02/mmlu_college_medicine.yaml | 6 - .../style_02/mmlu_college_physics.yaml | 6 - .../style_02/mmlu_computer_security.yaml | 6 - .../style_02/mmlu_conceptual_physics.yaml | 6 - .../style_02/mmlu_econometrics.yaml | 6 - .../style_02/mmlu_electrical_engineering.yaml | 6 - .../style_02/mmlu_elementary_mathematics.yaml | 6 - .../style_02/mmlu_formal_logic.yaml | 6 - .../style_02/mmlu_global_facts.yaml | 6 - .../style_02/mmlu_high_school_biology.yaml | 6 - .../style_02/mmlu_high_school_chemistry.yaml | 6 - .../mmlu_high_school_computer_science.yaml | 6 - .../mmlu_high_school_european_history.yaml | 6 - .../style_02/mmlu_high_school_geography.yaml | 6 - ...u_high_school_government_and_politics.yaml | 6 - .../mmlu_high_school_macroeconomics.yaml | 6 - .../mmlu_high_school_mathematics.yaml | 6 - .../mmlu_high_school_microeconomics.yaml | 6 - .../style_02/mmlu_high_school_physics.yaml | 6 - .../style_02/mmlu_high_school_psychology.yaml | 6 - .../style_02/mmlu_high_school_statistics.yaml | 6 - .../style_02/mmlu_high_school_us_history.yaml | 6 - .../mmlu_high_school_world_history.yaml | 6 - .../style_02/mmlu_human_aging.yaml | 6 - .../style_02/mmlu_human_sexuality.yaml | 6 - .../style_02/mmlu_international_law.yaml | 6 - .../style_02/mmlu_jurisprudence.yaml | 6 - .../style_02/mmlu_logical_fallacies.yaml | 6 - .../style_02/mmlu_machine_learning.yaml | 6 - .../style_02/mmlu_management.yaml | 6 - .../style_02/mmlu_marketing.yaml | 6 - .../style_02/mmlu_medical_genetics.yaml | 6 - .../style_02/mmlu_miscellaneous.yaml | 6 - .../style_02/mmlu_moral_disputes.yaml | 6 - .../style_02/mmlu_moral_scenarios.yaml | 6 - .../style_02/mmlu_nutrition.yaml | 6 - .../style_02/mmlu_philosophy.yaml | 6 - .../style_02/mmlu_prehistory.yaml | 6 - .../mmlu_professional_accounting.yaml | 6 - .../style_02/mmlu_professional_law.yaml | 6 - .../style_02/mmlu_professional_medicine.yaml | 6 - .../mmlu_professional_psychology.yaml | 6 - .../style_02/mmlu_public_relations.yaml | 6 - .../style_02/mmlu_security_studies.yaml | 6 - .../style_02/mmlu_sociology.yaml | 6 - .../style_02/mmlu_us_foreign_policy.yaml | 6 - .../style_02/mmlu_virology.yaml | 6 - .../style_02/mmlu_world_religions.yaml | 6 - .../style_03/_default_template_yaml | 13 -- .../prompt_variation/style_03/_mmlu.yaml | 6 - .../style_03/mmlu_abstract_algebra.yaml | 6 - .../style_03/mmlu_anatomy.yaml | 6 - .../style_03/mmlu_astronomy.yaml | 6 - .../style_03/mmlu_business_ethics.yaml | 6 - .../style_03/mmlu_clinical_knowledge.yaml | 6 - .../style_03/mmlu_college_biology.yaml | 6 - .../style_03/mmlu_college_chemistry.yaml | 6 - .../mmlu_college_computer_science.yaml | 6 - .../style_03/mmlu_college_mathematics.yaml | 6 - .../style_03/mmlu_college_medicine.yaml | 6 - .../style_03/mmlu_college_physics.yaml | 6 - .../style_03/mmlu_computer_security.yaml | 6 - .../style_03/mmlu_conceptual_physics.yaml | 6 - .../style_03/mmlu_econometrics.yaml | 6 - .../style_03/mmlu_electrical_engineering.yaml | 6 - .../style_03/mmlu_elementary_mathematics.yaml | 6 - .../style_03/mmlu_formal_logic.yaml | 6 - .../style_03/mmlu_global_facts.yaml | 6 - .../style_03/mmlu_high_school_biology.yaml | 6 - .../style_03/mmlu_high_school_chemistry.yaml | 6 - .../mmlu_high_school_computer_science.yaml | 6 - .../mmlu_high_school_european_history.yaml | 6 - .../style_03/mmlu_high_school_geography.yaml | 6 - ...u_high_school_government_and_politics.yaml | 6 - .../mmlu_high_school_macroeconomics.yaml | 6 - .../mmlu_high_school_mathematics.yaml | 6 - .../mmlu_high_school_microeconomics.yaml | 6 - .../style_03/mmlu_high_school_physics.yaml | 6 - .../style_03/mmlu_high_school_psychology.yaml | 6 - .../style_03/mmlu_high_school_statistics.yaml | 6 - .../style_03/mmlu_high_school_us_history.yaml | 6 - .../mmlu_high_school_world_history.yaml | 6 - .../style_03/mmlu_human_aging.yaml | 6 - .../style_03/mmlu_human_sexuality.yaml | 6 - .../style_03/mmlu_international_law.yaml | 6 - .../style_03/mmlu_jurisprudence.yaml | 6 - .../style_03/mmlu_logical_fallacies.yaml | 6 - .../style_03/mmlu_machine_learning.yaml | 6 - .../style_03/mmlu_management.yaml | 6 - .../style_03/mmlu_marketing.yaml | 6 - .../style_03/mmlu_medical_genetics.yaml | 6 - .../style_03/mmlu_miscellaneous.yaml | 6 - .../style_03/mmlu_moral_disputes.yaml | 6 - .../style_03/mmlu_moral_scenarios.yaml | 6 - .../style_03/mmlu_nutrition.yaml | 6 - .../style_03/mmlu_philosophy.yaml | 6 - .../style_03/mmlu_prehistory.yaml | 6 - .../mmlu_professional_accounting.yaml | 6 - .../style_03/mmlu_professional_law.yaml | 6 - .../style_03/mmlu_professional_medicine.yaml | 6 - .../mmlu_professional_psychology.yaml | 6 - .../style_03/mmlu_public_relations.yaml | 6 - .../style_03/mmlu_security_studies.yaml | 6 - .../style_03/mmlu_sociology.yaml | 6 - .../style_03/mmlu_us_foreign_policy.yaml | 6 - .../style_03/mmlu_virology.yaml | 6 - .../style_03/mmlu_world_religions.yaml | 6 - .../tasks/mmlu/alternative_worlds/styles.py | 114 ------------------ .../tasks/piqa/alternative_worlds/README.md | 20 --- .../output_variation/_piqa_alt_ov_yaml | 20 --- .../output_variation/piqa_alt_ov.yaml | 10 -- .../output_variation/style_01/a.yaml | 6 - .../output_variation/style_01/b.yaml | 6 - .../output_variation/style_01/c.yaml | 6 - .../output_variation/style_02/a.yaml | 6 - .../output_variation/style_02/b.yaml | 6 - .../output_variation/style_02/c.yaml | 6 - .../output_variation/style_03/a.yaml | 6 - .../output_variation/style_03/b.yaml | 6 - .../output_variation/style_03/c.yaml | 6 - .../output_variation/style_04/a.yaml | 6 - .../output_variation/style_04/b.yaml | 6 - .../output_variation/style_04/c.yaml | 6 - .../output_variation/style_05/a.yaml | 6 - .../output_variation/style_05/b.yaml | 6 - .../output_variation/style_05/c.yaml | 6 - .../output_variation/style_06/a.yaml | 6 - .../output_variation/style_06/b.yaml | 6 - .../output_variation/style_06/c.yaml | 6 - .../output_variation/style_07/a.yaml | 6 - .../output_variation/style_07/b.yaml | 6 - .../output_variation/style_07/c.yaml | 6 - .../output_variation/style_08/a.yaml | 6 - .../output_variation/style_08/b.yaml | 6 - .../output_variation/style_08/c.yaml | 6 - .../output_variation/styles.py | 90 -------------- .../prompt_variation/_piqa_yaml | 22 ---- .../prompt_variation/style_01.yaml | 5 - .../prompt_variation/style_02.yaml | 5 - .../prompt_variation/style_03.yaml | 5 - .../output_variation/_sciq_alt_yaml | 23 ---- .../output_variation/sciq_alt_ov.yaml | 10 -- .../output_variation/style_01/a.yaml | 5 - .../output_variation/style_01/b.yaml | 5 - .../output_variation/style_01/c.yaml | 5 - .../output_variation/style_02/a.yaml | 5 - .../output_variation/style_02/b.yaml | 5 - .../output_variation/style_02/c.yaml | 5 - .../output_variation/style_03/a.yaml | 5 - .../output_variation/style_03/b.yaml | 5 - .../output_variation/style_03/c.yaml | 5 - .../output_variation/style_04/a.yaml | 5 - .../output_variation/style_04/b.yaml | 5 - .../output_variation/style_04/c.yaml | 5 - .../output_variation/style_05/a.yaml | 5 - .../output_variation/style_05/b.yaml | 5 - .../output_variation/style_05/c.yaml | 5 - .../output_variation/style_06/a.yaml | 5 - .../output_variation/style_06/b.yaml | 5 - .../output_variation/style_06/c.yaml | 5 - .../output_variation/style_07/a.yaml | 5 - .../output_variation/style_07/b.yaml | 5 - .../output_variation/style_07/c.yaml | 5 - .../output_variation/style_08/a.yaml | 5 - .../output_variation/style_08/b.yaml | 5 - .../output_variation/style_08/c.yaml | 5 - .../output_variation/styles.py | 89 -------------- .../prompt_variation/style_01.yaml | 17 --- .../prompt_variation/style_02.yaml | 17 --- .../prompt_variation/style_03.yaml | 17 --- .../tasks/siqa/alternative_worlds/README.md | 20 --- .../output_variation/_siqa_alt_yaml | 19 --- .../output_variation/siqa_alt.yaml | 10 -- .../output_variation/style_01/a.yaml | 6 - .../output_variation/style_01/b.yaml | 6 - .../output_variation/style_01/c.yaml | 6 - .../output_variation/style_02/a.yaml | 6 - .../output_variation/style_02/b.yaml | 6 - .../output_variation/style_02/c.yaml | 6 - .../output_variation/style_03/a.yaml | 6 - .../output_variation/style_03/b.yaml | 6 - .../output_variation/style_03/c.yaml | 6 - .../output_variation/style_04/a.yaml | 6 - .../output_variation/style_04/b.yaml | 6 - .../output_variation/style_04/c.yaml | 6 - .../output_variation/style_05/a.yaml | 6 - .../output_variation/style_05/b.yaml | 6 - .../output_variation/style_05/c.yaml | 6 - .../output_variation/style_06/a.yaml | 6 - .../output_variation/style_06/b.yaml | 6 - .../output_variation/style_06/c.yaml | 6 - .../output_variation/style_07/a.yaml | 6 - .../output_variation/style_07/b.yaml | 6 - .../output_variation/style_07/c.yaml | 6 - .../output_variation/style_08/a.yaml | 6 - .../output_variation/style_08/b.yaml | 6 - .../output_variation/style_08/c.yaml | 6 - .../output_variation/styles.py | 90 -------------- .../prompt_variation/_siqa_alt_yaml | 19 --- .../prompt_variation/style_01.yaml | 5 - .../prompt_variation/style_02.yaml | 5 - .../prompt_variation/style_03.yaml | 5 - 2025 files changed, 13284 deletions(-) delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/README.md delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/_arc_challenge_alt_yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/arc_challenge_alt.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_01/a.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_01/b.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_01/c.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_02/a.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_02/b.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_02/c.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_03/a.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_03/b.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_03/c.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_04/a.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_04/b.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_04/c.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_05/a.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_05/b.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_05/c.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_06/a.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_06/b.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_06/c.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_07/a.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_07/b.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_07/c.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_08/a.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_08/b.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_08/c.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/styles.py delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/prompt_variation/_arc_challenge_alt_yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/prompt_variation/style_01.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/prompt_variation/style_02.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_challenge/prompt_variation/style_03.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/README.md delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/_arc_easy_alt_yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/arc_easy_alt.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_01/a.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_01/b.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_01/c.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_02/a.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_02/b.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_02/c.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_03/a.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_03/b.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_03/c.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_04/a.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_04/b.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_04/c.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_05/a.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_05/b.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_05/c.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_06/a.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_06/b.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_06/c.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_07/a.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_07/b.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_07/c.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_08/a.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_08/b.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_08/c.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/styles.py delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/prompt_variation/_arc_easy_alt_yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/prompt_variation/style_01.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/prompt_variation/style_02.yaml delete mode 100644 lm_eval/tasks/arc/alternative_worlds/arc_easy/prompt_variation/style_03.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/_template_yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/arithmetic_alt.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_00/_template_00_yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_1dc.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_2da.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_2dm.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_2ds.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_3da.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_3ds.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_4da.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_4ds.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_5da.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_5ds.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_01/_template_01_yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_1dc.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_2da.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_2dm.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_2ds.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_3da.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_3ds.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_4da.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_4ds.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_5da.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_5ds.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_02/_template_02_yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_1dc.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_2da.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_2dm.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_2ds.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_3da.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_3ds.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_4da.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_4ds.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_5da.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_5ds.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_03/_template_03_yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_1dc.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_2da.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_2dm.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_2ds.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_3da.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_3ds.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_4da.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_4ds.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_5da.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_5ds.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_04/_template_04_yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_1dc.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_2da.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_2dm.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_2ds.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_3da.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_3ds.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_4da.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_4ds.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_5da.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_5ds.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_05/_template_05_yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_1dc.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_2da.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_2dm.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_2ds.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_3da.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_3ds.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_4da.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_4ds.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_5da.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_5ds.yaml delete mode 100644 lm_eval/tasks/arithmetic/alternative_worlds/utils.py delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/README.md delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/bbh_alt_pv_zeroshot.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/_zeroshot_template_yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/boolean_expressions.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/causal_judgement.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/date_understanding.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/disambiguation_qa.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/formal_fallacies.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/geometric_shapes.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/hyperbaton.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/logical_deduction_five_objects.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/logical_deduction_seven_objects.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/logical_deduction_three_objects.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/movie_recommendation.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/navigate.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/penguins_in_a_table.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/reasoning_about_colored_objects.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/ruin_names.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/salient_translation_error_detection.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/snarks.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/sports_understanding.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/temporal_sequences.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/tracking_shuffled_objects_five_objects.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/tracking_shuffled_objects_seven_objects.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/tracking_shuffled_objects_three_objects.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/web_of_lies.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/_zeroshot_template_yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/boolean_expressions.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/causal_judgement.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/date_understanding.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/disambiguation_qa.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/formal_fallacies.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/geometric_shapes.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/hyperbaton.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/logical_deduction_five_objects.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/logical_deduction_seven_objects.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/logical_deduction_three_objects.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/movie_recommendation.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/navigate.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/penguins_in_a_table.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/reasoning_about_colored_objects.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/ruin_names.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/salient_translation_error_detection.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/snarks.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/sports_understanding.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/temporal_sequences.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/tracking_shuffled_objects_five_objects.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/tracking_shuffled_objects_seven_objects.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/tracking_shuffled_objects_three_objects.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/web_of_lies.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/_zeroshot_template_yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/boolean_expressions.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/causal_judgement.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/date_understanding.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/disambiguation_qa.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/formal_fallacies.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/geometric_shapes.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/hyperbaton.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/logical_deduction_five_objects.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/logical_deduction_seven_objects.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/logical_deduction_three_objects.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/movie_recommendation.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/navigate.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/penguins_in_a_table.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/reasoning_about_colored_objects.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/ruin_names.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/salient_translation_error_detection.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/snarks.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/sports_understanding.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/temporal_sequences.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/tracking_shuffled_objects_five_objects.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/tracking_shuffled_objects_seven_objects.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/tracking_shuffled_objects_three_objects.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/web_of_lies.yaml delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/styles.py delete mode 100644 lm_eval/tasks/bbh/alternative_worlds/prompt_variation/utils.py delete mode 100644 lm_eval/tasks/hellaswag/alternative_worlds/README.md delete mode 100644 lm_eval/tasks/hellaswag/alternative_worlds/_hellaswag_alt_yaml delete mode 100644 lm_eval/tasks/hellaswag/alternative_worlds/hellaswag_alt.yaml delete mode 100644 lm_eval/tasks/hellaswag/alternative_worlds/style_01/a.yaml delete mode 100644 lm_eval/tasks/hellaswag/alternative_worlds/style_01/b.yaml delete mode 100644 lm_eval/tasks/hellaswag/alternative_worlds/style_01/c.yaml delete mode 100644 lm_eval/tasks/hellaswag/alternative_worlds/style_02/a.yaml delete mode 100644 lm_eval/tasks/hellaswag/alternative_worlds/style_02/b.yaml delete mode 100644 lm_eval/tasks/hellaswag/alternative_worlds/style_02/c.yaml delete mode 100644 lm_eval/tasks/hellaswag/alternative_worlds/style_03/a.yaml delete mode 100644 lm_eval/tasks/hellaswag/alternative_worlds/style_03/b.yaml delete mode 100644 lm_eval/tasks/hellaswag/alternative_worlds/style_03/c.yaml delete mode 100644 lm_eval/tasks/hellaswag/alternative_worlds/style_04/a.yaml delete mode 100644 lm_eval/tasks/hellaswag/alternative_worlds/style_04/b.yaml delete mode 100644 lm_eval/tasks/hellaswag/alternative_worlds/style_04/c.yaml delete mode 100644 lm_eval/tasks/hellaswag/alternative_worlds/style_05/a.yaml delete mode 100644 lm_eval/tasks/hellaswag/alternative_worlds/style_05/b.yaml delete mode 100644 lm_eval/tasks/hellaswag/alternative_worlds/style_05/c.yaml delete mode 100644 lm_eval/tasks/hellaswag/alternative_worlds/style_06/a.yaml delete mode 100644 lm_eval/tasks/hellaswag/alternative_worlds/style_06/b.yaml delete mode 100644 lm_eval/tasks/hellaswag/alternative_worlds/style_06/c.yaml delete mode 100644 lm_eval/tasks/hellaswag/alternative_worlds/style_07/a.yaml delete mode 100644 lm_eval/tasks/hellaswag/alternative_worlds/style_07/b.yaml delete mode 100644 lm_eval/tasks/hellaswag/alternative_worlds/style_07/c.yaml delete mode 100644 lm_eval/tasks/hellaswag/alternative_worlds/style_08/a.yaml delete mode 100644 lm_eval/tasks/hellaswag/alternative_worlds/style_08/b.yaml delete mode 100644 lm_eval/tasks/hellaswag/alternative_worlds/style_08/c.yaml delete mode 100644 lm_eval/tasks/hellaswag/alternative_worlds/styles.py delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/output_variation/README.md delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/output_variation/_mathqa_alt_yaml delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/output_variation/mathqa_alt.yaml delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_01/a.yaml delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_01/b.yaml delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_01/c.yaml delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_02/a.yaml delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_02/b.yaml delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_02/c.yaml delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_03/a.yaml delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_03/b.yaml delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_03/c.yaml delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_04/a.yaml delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_04/b.yaml delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_04/c.yaml delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_05/a.yaml delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_05/b.yaml delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_05/c.yaml delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_06/a.yaml delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_06/b.yaml delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_06/c.yaml delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_07/a.yaml delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_07/b.yaml delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_07/c.yaml delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_08/a.yaml delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_08/b.yaml delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_08/c.yaml delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/output_variation/styles.py delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/prompt_variation/style_01.yaml delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/prompt_variation/style_02.yaml delete mode 100644 lm_eval/tasks/mathqa/alternative_worlds/prompt_variation/style_03.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/mmlu_output_variation.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/mmlu_prompt_variation.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/_mmlu_ov_01.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/_mmlu.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/_template_yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_abstract_algebra.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_anatomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_astronomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_business_ethics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_clinical_knowledge.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_college_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_college_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_college_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_college_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_college_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_college_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_computer_security.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_conceptual_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_econometrics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_electrical_engineering.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_elementary_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_formal_logic.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_global_facts.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_european_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_geography.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_government_and_politics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_macroeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_microeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_statistics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_us_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_world_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_human_aging.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_human_sexuality.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_international_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_jurisprudence.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_logical_fallacies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_machine_learning.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_management.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_marketing.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_medical_genetics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_miscellaneous.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_moral_disputes.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_moral_scenarios.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_nutrition.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_philosophy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_prehistory.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_professional_accounting.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_professional_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_professional_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_professional_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_public_relations.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_security_studies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_sociology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_us_foreign_policy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_virology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_world_religions.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/_mmlu.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/_template_yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_abstract_algebra.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_anatomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_astronomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_business_ethics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_clinical_knowledge.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_college_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_college_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_college_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_college_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_college_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_college_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_computer_security.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_conceptual_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_econometrics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_electrical_engineering.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_elementary_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_formal_logic.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_global_facts.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_european_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_geography.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_government_and_politics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_macroeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_microeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_statistics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_us_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_world_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_human_aging.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_human_sexuality.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_international_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_jurisprudence.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_logical_fallacies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_machine_learning.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_management.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_marketing.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_medical_genetics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_miscellaneous.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_moral_disputes.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_moral_scenarios.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_nutrition.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_philosophy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_prehistory.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_professional_accounting.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_professional_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_professional_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_professional_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_public_relations.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_security_studies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_sociology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_us_foreign_policy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_virology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_world_religions.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/_mmlu.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/_template_yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_abstract_algebra.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_anatomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_astronomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_business_ethics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_clinical_knowledge.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_college_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_college_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_college_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_college_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_college_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_college_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_computer_security.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_conceptual_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_econometrics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_electrical_engineering.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_elementary_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_formal_logic.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_global_facts.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_european_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_geography.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_government_and_politics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_macroeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_microeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_statistics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_us_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_world_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_human_aging.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_human_sexuality.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_international_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_jurisprudence.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_logical_fallacies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_machine_learning.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_management.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_marketing.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_medical_genetics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_miscellaneous.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_moral_disputes.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_moral_scenarios.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_nutrition.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_philosophy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_prehistory.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_professional_accounting.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_professional_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_professional_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_professional_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_public_relations.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_security_studies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_sociology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_us_foreign_policy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_virology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_world_religions.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/_mmlu_ov_02.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/_mmlu.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/_template_yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_abstract_algebra.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_anatomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_astronomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_business_ethics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_clinical_knowledge.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_college_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_college_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_college_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_college_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_college_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_college_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_computer_security.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_conceptual_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_econometrics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_electrical_engineering.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_elementary_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_formal_logic.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_global_facts.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_european_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_geography.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_government_and_politics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_macroeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_microeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_statistics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_us_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_world_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_human_aging.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_human_sexuality.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_international_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_jurisprudence.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_logical_fallacies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_machine_learning.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_management.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_marketing.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_medical_genetics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_miscellaneous.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_moral_disputes.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_moral_scenarios.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_nutrition.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_philosophy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_prehistory.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_professional_accounting.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_professional_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_professional_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_professional_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_public_relations.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_security_studies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_sociology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_us_foreign_policy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_virology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_world_religions.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/_mmlu.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/_template_yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_abstract_algebra.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_anatomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_astronomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_business_ethics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_clinical_knowledge.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_college_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_college_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_college_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_college_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_college_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_college_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_computer_security.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_conceptual_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_econometrics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_electrical_engineering.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_elementary_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_formal_logic.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_global_facts.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_european_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_geography.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_government_and_politics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_macroeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_microeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_statistics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_us_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_world_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_human_aging.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_human_sexuality.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_international_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_jurisprudence.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_logical_fallacies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_machine_learning.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_management.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_marketing.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_medical_genetics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_miscellaneous.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_moral_disputes.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_moral_scenarios.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_nutrition.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_philosophy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_prehistory.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_professional_accounting.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_professional_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_professional_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_professional_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_public_relations.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_security_studies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_sociology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_us_foreign_policy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_virology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_world_religions.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/_mmlu.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/_template_yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_abstract_algebra.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_anatomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_astronomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_business_ethics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_clinical_knowledge.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_college_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_college_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_college_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_college_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_college_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_college_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_computer_security.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_conceptual_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_econometrics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_electrical_engineering.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_elementary_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_formal_logic.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_global_facts.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_european_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_geography.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_government_and_politics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_macroeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_microeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_statistics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_us_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_world_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_human_aging.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_human_sexuality.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_international_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_jurisprudence.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_logical_fallacies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_machine_learning.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_management.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_marketing.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_medical_genetics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_miscellaneous.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_moral_disputes.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_moral_scenarios.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_nutrition.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_philosophy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_prehistory.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_professional_accounting.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_professional_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_professional_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_professional_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_public_relations.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_security_studies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_sociology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_us_foreign_policy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_virology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_world_religions.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/_mmlu_ov_03.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/_mmlu.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/_template_yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_abstract_algebra.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_anatomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_astronomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_business_ethics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_clinical_knowledge.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_college_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_college_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_college_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_college_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_college_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_college_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_computer_security.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_conceptual_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_econometrics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_electrical_engineering.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_elementary_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_formal_logic.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_global_facts.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_european_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_geography.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_government_and_politics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_macroeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_microeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_statistics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_us_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_world_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_human_aging.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_human_sexuality.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_international_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_jurisprudence.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_logical_fallacies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_machine_learning.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_management.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_marketing.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_medical_genetics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_miscellaneous.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_moral_disputes.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_moral_scenarios.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_nutrition.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_philosophy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_prehistory.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_professional_accounting.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_professional_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_professional_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_professional_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_public_relations.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_security_studies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_sociology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_us_foreign_policy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_virology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_world_religions.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/_mmlu.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/_template_yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_abstract_algebra.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_anatomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_astronomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_business_ethics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_clinical_knowledge.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_college_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_college_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_college_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_college_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_college_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_college_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_computer_security.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_conceptual_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_econometrics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_electrical_engineering.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_elementary_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_formal_logic.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_global_facts.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_european_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_geography.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_government_and_politics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_macroeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_microeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_statistics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_us_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_world_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_human_aging.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_human_sexuality.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_international_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_jurisprudence.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_logical_fallacies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_machine_learning.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_management.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_marketing.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_medical_genetics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_miscellaneous.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_moral_disputes.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_moral_scenarios.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_nutrition.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_philosophy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_prehistory.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_professional_accounting.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_professional_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_professional_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_professional_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_public_relations.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_security_studies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_sociology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_us_foreign_policy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_virology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_world_religions.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/_mmlu.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/_template_yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_abstract_algebra.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_anatomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_astronomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_business_ethics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_clinical_knowledge.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_college_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_college_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_college_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_college_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_college_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_college_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_computer_security.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_conceptual_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_econometrics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_electrical_engineering.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_elementary_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_formal_logic.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_global_facts.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_european_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_geography.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_government_and_politics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_macroeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_microeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_statistics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_us_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_world_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_human_aging.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_human_sexuality.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_international_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_jurisprudence.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_logical_fallacies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_machine_learning.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_management.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_marketing.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_medical_genetics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_miscellaneous.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_moral_disputes.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_moral_scenarios.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_nutrition.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_philosophy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_prehistory.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_professional_accounting.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_professional_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_professional_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_professional_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_public_relations.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_security_studies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_sociology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_us_foreign_policy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_virology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_world_religions.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/_mmlu_ov_04.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/_mmlu.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/_template_yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_abstract_algebra.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_anatomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_astronomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_business_ethics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_clinical_knowledge.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_college_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_college_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_college_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_college_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_college_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_college_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_computer_security.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_conceptual_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_econometrics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_electrical_engineering.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_elementary_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_formal_logic.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_global_facts.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_european_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_geography.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_government_and_politics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_macroeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_microeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_statistics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_us_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_world_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_human_aging.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_human_sexuality.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_international_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_jurisprudence.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_logical_fallacies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_machine_learning.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_management.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_marketing.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_medical_genetics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_miscellaneous.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_moral_disputes.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_moral_scenarios.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_nutrition.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_philosophy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_prehistory.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_professional_accounting.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_professional_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_professional_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_professional_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_public_relations.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_security_studies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_sociology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_us_foreign_policy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_virology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_world_religions.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/_mmlu.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/_template_yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_abstract_algebra.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_anatomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_astronomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_business_ethics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_clinical_knowledge.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_college_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_college_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_college_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_college_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_college_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_college_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_computer_security.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_conceptual_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_econometrics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_electrical_engineering.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_elementary_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_formal_logic.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_global_facts.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_european_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_geography.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_government_and_politics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_macroeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_microeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_statistics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_us_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_world_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_human_aging.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_human_sexuality.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_international_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_jurisprudence.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_logical_fallacies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_machine_learning.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_management.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_marketing.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_medical_genetics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_miscellaneous.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_moral_disputes.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_moral_scenarios.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_nutrition.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_philosophy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_prehistory.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_professional_accounting.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_professional_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_professional_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_professional_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_public_relations.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_security_studies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_sociology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_us_foreign_policy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_virology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_world_religions.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/_mmlu.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/_template_yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_abstract_algebra.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_anatomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_astronomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_business_ethics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_clinical_knowledge.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_college_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_college_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_college_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_college_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_college_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_college_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_computer_security.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_conceptual_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_econometrics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_electrical_engineering.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_elementary_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_formal_logic.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_global_facts.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_european_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_geography.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_government_and_politics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_macroeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_microeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_statistics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_us_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_world_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_human_aging.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_human_sexuality.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_international_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_jurisprudence.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_logical_fallacies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_machine_learning.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_management.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_marketing.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_medical_genetics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_miscellaneous.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_moral_disputes.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_moral_scenarios.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_nutrition.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_philosophy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_prehistory.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_professional_accounting.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_professional_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_professional_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_professional_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_public_relations.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_security_studies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_sociology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_us_foreign_policy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_virology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_world_religions.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/_mmlu_ov_05.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/_mmlu.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/_template_yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_abstract_algebra.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_anatomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_astronomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_business_ethics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_clinical_knowledge.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_college_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_college_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_college_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_college_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_college_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_college_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_computer_security.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_conceptual_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_econometrics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_electrical_engineering.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_elementary_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_formal_logic.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_global_facts.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_european_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_geography.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_government_and_politics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_macroeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_microeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_statistics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_us_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_world_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_human_aging.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_human_sexuality.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_international_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_jurisprudence.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_logical_fallacies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_machine_learning.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_management.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_marketing.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_medical_genetics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_miscellaneous.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_moral_disputes.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_moral_scenarios.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_nutrition.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_philosophy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_prehistory.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_professional_accounting.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_professional_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_professional_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_professional_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_public_relations.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_security_studies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_sociology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_us_foreign_policy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_virology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_world_religions.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/_mmlu.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/_template_yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_abstract_algebra.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_anatomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_astronomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_business_ethics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_clinical_knowledge.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_college_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_college_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_college_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_college_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_college_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_college_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_computer_security.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_conceptual_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_econometrics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_electrical_engineering.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_elementary_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_formal_logic.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_global_facts.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_european_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_geography.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_government_and_politics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_macroeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_microeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_statistics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_us_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_world_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_human_aging.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_human_sexuality.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_international_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_jurisprudence.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_logical_fallacies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_machine_learning.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_management.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_marketing.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_medical_genetics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_miscellaneous.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_moral_disputes.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_moral_scenarios.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_nutrition.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_philosophy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_prehistory.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_professional_accounting.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_professional_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_professional_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_professional_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_public_relations.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_security_studies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_sociology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_us_foreign_policy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_virology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_world_religions.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/_mmlu.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/_template_yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_abstract_algebra.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_anatomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_astronomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_business_ethics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_clinical_knowledge.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_college_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_college_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_college_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_college_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_college_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_college_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_computer_security.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_conceptual_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_econometrics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_electrical_engineering.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_elementary_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_formal_logic.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_global_facts.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_european_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_geography.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_government_and_politics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_macroeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_microeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_statistics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_us_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_world_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_human_aging.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_human_sexuality.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_international_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_jurisprudence.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_logical_fallacies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_machine_learning.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_management.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_marketing.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_medical_genetics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_miscellaneous.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_moral_disputes.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_moral_scenarios.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_nutrition.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_philosophy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_prehistory.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_professional_accounting.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_professional_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_professional_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_professional_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_public_relations.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_security_studies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_sociology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_us_foreign_policy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_virology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_world_religions.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/_mmlu.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/_template_yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_abstract_algebra.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_anatomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_astronomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_business_ethics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_clinical_knowledge.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_college_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_college_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_college_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_college_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_college_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_college_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_computer_security.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_conceptual_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_econometrics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_electrical_engineering.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_elementary_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_formal_logic.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_global_facts.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_european_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_geography.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_government_and_politics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_macroeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_microeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_statistics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_us_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_world_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_human_aging.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_human_sexuality.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_international_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_jurisprudence.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_logical_fallacies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_machine_learning.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_management.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_marketing.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_medical_genetics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_miscellaneous.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_moral_disputes.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_moral_scenarios.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_nutrition.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_philosophy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_prehistory.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_professional_accounting.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_professional_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_professional_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_professional_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_public_relations.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_security_studies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_sociology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_us_foreign_policy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_virology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_world_religions.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/_mmlu_ov_06.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/_mmlu.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/_template_yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_abstract_algebra.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_anatomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_astronomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_business_ethics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_clinical_knowledge.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_college_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_college_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_college_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_college_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_college_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_college_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_computer_security.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_conceptual_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_econometrics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_electrical_engineering.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_elementary_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_formal_logic.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_global_facts.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_european_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_geography.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_government_and_politics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_macroeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_microeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_statistics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_us_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_world_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_human_aging.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_human_sexuality.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_international_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_jurisprudence.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_logical_fallacies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_machine_learning.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_management.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_marketing.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_medical_genetics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_miscellaneous.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_moral_disputes.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_moral_scenarios.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_nutrition.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_philosophy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_prehistory.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_professional_accounting.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_professional_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_professional_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_professional_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_public_relations.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_security_studies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_sociology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_us_foreign_policy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_virology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_world_religions.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/_mmlu.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/_template_yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_abstract_algebra.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_anatomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_astronomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_business_ethics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_clinical_knowledge.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_college_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_college_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_college_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_college_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_college_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_college_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_computer_security.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_conceptual_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_econometrics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_electrical_engineering.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_elementary_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_formal_logic.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_global_facts.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_european_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_geography.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_government_and_politics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_macroeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_microeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_statistics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_us_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_world_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_human_aging.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_human_sexuality.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_international_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_jurisprudence.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_logical_fallacies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_machine_learning.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_management.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_marketing.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_medical_genetics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_miscellaneous.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_moral_disputes.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_moral_scenarios.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_nutrition.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_philosophy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_prehistory.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_professional_accounting.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_professional_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_professional_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_professional_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_public_relations.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_security_studies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_sociology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_us_foreign_policy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_virology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_world_religions.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/_mmlu.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/_template_yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_abstract_algebra.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_anatomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_astronomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_business_ethics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_clinical_knowledge.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_college_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_college_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_college_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_college_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_college_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_college_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_computer_security.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_conceptual_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_econometrics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_electrical_engineering.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_elementary_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_formal_logic.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_global_facts.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_european_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_geography.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_government_and_politics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_macroeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_microeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_statistics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_us_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_world_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_human_aging.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_human_sexuality.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_international_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_jurisprudence.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_logical_fallacies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_machine_learning.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_management.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_marketing.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_medical_genetics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_miscellaneous.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_moral_disputes.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_moral_scenarios.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_nutrition.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_philosophy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_prehistory.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_professional_accounting.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_professional_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_professional_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_professional_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_public_relations.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_security_studies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_sociology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_us_foreign_policy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_virology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_world_religions.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/_mmlu_ov_07.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/_mmlu.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/_template_yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_abstract_algebra.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_anatomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_astronomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_business_ethics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_clinical_knowledge.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_college_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_college_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_college_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_college_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_college_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_college_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_computer_security.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_conceptual_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_econometrics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_electrical_engineering.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_elementary_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_formal_logic.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_global_facts.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_european_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_geography.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_government_and_politics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_macroeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_microeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_statistics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_us_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_world_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_human_aging.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_human_sexuality.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_international_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_jurisprudence.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_logical_fallacies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_machine_learning.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_management.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_marketing.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_medical_genetics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_miscellaneous.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_moral_disputes.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_moral_scenarios.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_nutrition.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_philosophy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_prehistory.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_professional_accounting.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_professional_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_professional_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_professional_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_public_relations.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_security_studies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_sociology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_us_foreign_policy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_virology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_world_religions.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/_mmlu.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/_template_yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_abstract_algebra.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_anatomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_astronomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_business_ethics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_clinical_knowledge.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_college_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_college_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_college_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_college_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_college_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_college_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_computer_security.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_conceptual_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_econometrics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_electrical_engineering.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_elementary_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_formal_logic.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_global_facts.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_european_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_geography.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_government_and_politics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_macroeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_microeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_statistics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_us_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_world_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_human_aging.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_human_sexuality.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_international_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_jurisprudence.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_logical_fallacies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_machine_learning.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_management.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_marketing.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_medical_genetics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_miscellaneous.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_moral_disputes.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_moral_scenarios.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_nutrition.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_philosophy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_prehistory.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_professional_accounting.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_professional_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_professional_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_professional_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_public_relations.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_security_studies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_sociology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_us_foreign_policy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_virology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_world_religions.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/_mmlu.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/_template_yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_abstract_algebra.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_anatomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_astronomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_business_ethics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_clinical_knowledge.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_college_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_college_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_college_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_college_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_college_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_college_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_computer_security.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_conceptual_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_econometrics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_electrical_engineering.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_elementary_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_formal_logic.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_global_facts.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_european_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_geography.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_government_and_politics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_macroeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_microeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_statistics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_us_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_world_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_human_aging.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_human_sexuality.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_international_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_jurisprudence.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_logical_fallacies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_machine_learning.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_management.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_marketing.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_medical_genetics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_miscellaneous.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_moral_disputes.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_moral_scenarios.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_nutrition.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_philosophy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_prehistory.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_professional_accounting.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_professional_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_professional_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_professional_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_public_relations.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_security_studies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_sociology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_us_foreign_policy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_virology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_world_religions.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/_mmlu_ov_08.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/_mmlu.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/_template_yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_abstract_algebra.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_anatomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_astronomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_business_ethics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_clinical_knowledge.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_college_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_college_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_college_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_college_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_college_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_college_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_computer_security.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_conceptual_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_econometrics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_electrical_engineering.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_elementary_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_formal_logic.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_global_facts.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_european_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_geography.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_government_and_politics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_macroeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_microeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_statistics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_us_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_world_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_human_aging.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_human_sexuality.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_international_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_jurisprudence.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_logical_fallacies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_machine_learning.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_management.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_marketing.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_medical_genetics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_miscellaneous.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_moral_disputes.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_moral_scenarios.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_nutrition.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_philosophy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_prehistory.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_professional_accounting.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_professional_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_professional_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_professional_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_public_relations.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_security_studies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_sociology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_us_foreign_policy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_virology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_world_religions.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/_mmlu.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/_template_yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_abstract_algebra.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_anatomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_astronomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_business_ethics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_clinical_knowledge.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_college_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_college_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_college_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_college_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_college_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_college_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_computer_security.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_conceptual_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_econometrics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_electrical_engineering.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_elementary_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_formal_logic.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_global_facts.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_european_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_geography.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_government_and_politics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_macroeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_microeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_statistics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_us_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_world_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_human_aging.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_human_sexuality.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_international_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_jurisprudence.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_logical_fallacies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_machine_learning.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_management.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_marketing.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_medical_genetics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_miscellaneous.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_moral_disputes.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_moral_scenarios.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_nutrition.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_philosophy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_prehistory.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_professional_accounting.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_professional_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_professional_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_professional_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_public_relations.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_security_studies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_sociology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_us_foreign_policy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_virology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_world_religions.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/_mmlu.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/_template_yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_abstract_algebra.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_anatomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_astronomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_business_ethics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_clinical_knowledge.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_college_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_college_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_college_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_college_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_college_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_college_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_computer_security.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_conceptual_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_econometrics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_electrical_engineering.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_elementary_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_formal_logic.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_global_facts.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_european_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_geography.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_government_and_politics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_macroeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_microeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_statistics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_us_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_world_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_human_aging.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_human_sexuality.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_international_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_jurisprudence.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_logical_fallacies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_machine_learning.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_management.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_marketing.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_medical_genetics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_miscellaneous.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_moral_disputes.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_moral_scenarios.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_nutrition.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_philosophy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_prehistory.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_professional_accounting.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_professional_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_professional_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_professional_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_public_relations.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_security_studies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_sociology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_us_foreign_policy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_virology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_world_religions.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/_default_template_yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/_mmlu.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_abstract_algebra.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_anatomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_astronomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_business_ethics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_clinical_knowledge.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_college_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_college_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_college_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_college_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_college_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_college_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_computer_security.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_conceptual_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_econometrics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_electrical_engineering.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_elementary_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_formal_logic.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_global_facts.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_european_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_geography.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_government_and_politics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_macroeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_microeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_statistics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_us_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_world_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_human_aging.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_human_sexuality.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_international_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_jurisprudence.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_logical_fallacies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_machine_learning.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_management.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_marketing.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_medical_genetics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_miscellaneous.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_moral_disputes.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_moral_scenarios.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_nutrition.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_philosophy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_prehistory.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_professional_accounting.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_professional_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_professional_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_professional_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_public_relations.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_security_studies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_sociology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_us_foreign_policy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_virology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_world_religions.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/_default_template_yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/_mmlu.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_abstract_algebra.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_anatomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_astronomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_business_ethics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_clinical_knowledge.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_college_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_college_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_college_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_college_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_college_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_college_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_computer_security.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_conceptual_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_econometrics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_electrical_engineering.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_elementary_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_formal_logic.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_global_facts.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_european_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_geography.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_government_and_politics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_macroeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_microeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_statistics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_us_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_world_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_human_aging.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_human_sexuality.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_international_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_jurisprudence.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_logical_fallacies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_machine_learning.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_management.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_marketing.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_medical_genetics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_miscellaneous.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_moral_disputes.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_moral_scenarios.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_nutrition.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_philosophy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_prehistory.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_professional_accounting.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_professional_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_professional_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_professional_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_public_relations.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_security_studies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_sociology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_us_foreign_policy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_virology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_world_religions.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/_default_template_yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/_mmlu.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_abstract_algebra.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_anatomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_astronomy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_business_ethics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_clinical_knowledge.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_college_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_college_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_college_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_college_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_college_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_college_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_computer_security.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_conceptual_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_econometrics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_electrical_engineering.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_elementary_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_formal_logic.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_global_facts.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_biology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_chemistry.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_computer_science.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_european_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_geography.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_government_and_politics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_macroeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_mathematics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_microeconomics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_physics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_statistics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_us_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_world_history.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_human_aging.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_human_sexuality.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_international_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_jurisprudence.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_logical_fallacies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_machine_learning.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_management.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_marketing.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_medical_genetics.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_miscellaneous.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_moral_disputes.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_moral_scenarios.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_nutrition.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_philosophy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_prehistory.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_professional_accounting.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_professional_law.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_professional_medicine.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_professional_psychology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_public_relations.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_security_studies.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_sociology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_us_foreign_policy.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_virology.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_world_religions.yaml delete mode 100644 lm_eval/tasks/mmlu/alternative_worlds/styles.py delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/README.md delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/output_variation/_piqa_alt_ov_yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/output_variation/piqa_alt_ov.yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/output_variation/style_01/a.yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/output_variation/style_01/b.yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/output_variation/style_01/c.yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/output_variation/style_02/a.yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/output_variation/style_02/b.yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/output_variation/style_02/c.yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/output_variation/style_03/a.yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/output_variation/style_03/b.yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/output_variation/style_03/c.yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/output_variation/style_04/a.yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/output_variation/style_04/b.yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/output_variation/style_04/c.yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/output_variation/style_05/a.yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/output_variation/style_05/b.yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/output_variation/style_05/c.yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/output_variation/style_06/a.yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/output_variation/style_06/b.yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/output_variation/style_06/c.yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/output_variation/style_07/a.yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/output_variation/style_07/b.yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/output_variation/style_07/c.yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/output_variation/style_08/a.yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/output_variation/style_08/b.yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/output_variation/style_08/c.yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/output_variation/styles.py delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/prompt_variation/_piqa_yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/prompt_variation/style_01.yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/prompt_variation/style_02.yaml delete mode 100644 lm_eval/tasks/piqa/alternative_worlds/prompt_variation/style_03.yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/output_variation/_sciq_alt_yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/output_variation/sciq_alt_ov.yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/output_variation/style_01/a.yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/output_variation/style_01/b.yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/output_variation/style_01/c.yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/output_variation/style_02/a.yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/output_variation/style_02/b.yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/output_variation/style_02/c.yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/output_variation/style_03/a.yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/output_variation/style_03/b.yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/output_variation/style_03/c.yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/output_variation/style_04/a.yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/output_variation/style_04/b.yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/output_variation/style_04/c.yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/output_variation/style_05/a.yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/output_variation/style_05/b.yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/output_variation/style_05/c.yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/output_variation/style_06/a.yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/output_variation/style_06/b.yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/output_variation/style_06/c.yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/output_variation/style_07/a.yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/output_variation/style_07/b.yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/output_variation/style_07/c.yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/output_variation/style_08/a.yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/output_variation/style_08/b.yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/output_variation/style_08/c.yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/output_variation/styles.py delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/prompt_variation/style_01.yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/prompt_variation/style_02.yaml delete mode 100644 lm_eval/tasks/sciq/alternative_worlds/prompt_variation/style_03.yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/README.md delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/output_variation/_siqa_alt_yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/output_variation/siqa_alt.yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/output_variation/style_01/a.yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/output_variation/style_01/b.yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/output_variation/style_01/c.yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/output_variation/style_02/a.yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/output_variation/style_02/b.yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/output_variation/style_02/c.yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/output_variation/style_03/a.yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/output_variation/style_03/b.yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/output_variation/style_03/c.yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/output_variation/style_04/a.yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/output_variation/style_04/b.yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/output_variation/style_04/c.yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/output_variation/style_05/a.yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/output_variation/style_05/b.yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/output_variation/style_05/c.yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/output_variation/style_06/a.yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/output_variation/style_06/b.yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/output_variation/style_06/c.yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/output_variation/style_07/a.yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/output_variation/style_07/b.yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/output_variation/style_07/c.yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/output_variation/style_08/a.yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/output_variation/style_08/b.yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/output_variation/style_08/c.yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/output_variation/styles.py delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/prompt_variation/_siqa_alt_yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/prompt_variation/style_01.yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/prompt_variation/style_02.yaml delete mode 100644 lm_eval/tasks/siqa/alternative_worlds/prompt_variation/style_03.yaml diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/README.md b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/README.md deleted file mode 100644 index 93600ae1..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/README.md +++ /dev/null @@ -1,20 +0,0 @@ - - -Investigate affect of letter options -- (A) -- A) -- A. -- A\t -- (a) -- a) -- a. -- a\t - -Answer types: -- letters only - - original option - - just letter -- letters + continuation - - original option - - just letter -- continuation diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/_arc_challenge_alt_yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/_arc_challenge_alt_yaml deleted file mode 100644 index 6266a6fa..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/_arc_challenge_alt_yaml +++ /dev/null @@ -1,23 +0,0 @@ -group: - - ai2_arc -dataset_path: ai2_arc -dataset_name: ARC-Challenge -output_type: multiple_choice -training_split: train -validation_split: validation -test_split: test -doc_to_text: "Question: {{question}}\nAnswer:" -doc_to_target: "{{choices.label.index(answerKey)}}" -doc_to_choice: "{{choices.text}}" -should_decontaminate: true -doc_to_decontamination_query: "Question: {{question}}\nAnswer:" -metric_list: - - metric: acc - aggregation: mean - higher_is_better: true - - metric: acc_norm - aggregation: mean - higher_is_better: true - - metric: brier_score - aggregation: brier_score - higher_is_better: false diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/arc_challenge_alt.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/arc_challenge_alt.yaml deleted file mode 100644 index 128fec5f..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/arc_challenge_alt.yaml +++ /dev/null @@ -1,10 +0,0 @@ -group: arc_challenge_alt_ov -task: - - arc_challenge_alt_ov_01 - - arc_challenge_alt_ov_02 - - arc_challenge_alt_ov_03 - - arc_challenge_alt_ov_04 - - arc_challenge_alt_ov_05 - - arc_challenge_alt_ov_06 - - arc_challenge_alt_ov_07 - - arc_challenge_alt_ov_08 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_01/a.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_01/a.yaml deleted file mode 100644 index e2fe0438..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_01/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_challenge_alt_yaml -group: arc_challenge_alt_ov_01 -task: arc_challenge_alt_ov_01a -doc_to_text: !function ../styles.template_01 -doc_to_choice: !function ../styles.choice_01a -doc_to_decontamination_query: !function ../styles.template_01 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_01/b.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_01/b.yaml deleted file mode 100644 index f99170f6..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_01/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_challenge_alt_yaml -group: arc_challenge_alt_ov_01 -task: arc_challenge_alt_ov_01b -doc_to_text: !function ../styles.template_01 -doc_to_choice: !function ../styles.choice_01b -doc_to_decontamination_query: !function ../styles.template_01 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_01/c.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_01/c.yaml deleted file mode 100644 index f985df9e..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_01/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_challenge_alt_yaml -group: arc_challenge_alt_ov_01 -task: arc_challenge_alt_ov_01c -doc_to_text: !function ../styles.template_01 -doc_to_choice: !function ../styles.choice_01c -doc_to_decontamination_query: !function ../styles.template_01 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_02/a.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_02/a.yaml deleted file mode 100644 index b4ef10d2..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_02/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_challenge_alt_yaml -group: arc_challenge_alt_ov_02 -task: arc_challenge_alt_ov_02a -doc_to_text: !function ../styles.template_02 -doc_to_choice: !function ../styles.choice_02a -doc_to_decontamination_query: !function ../styles.template_02 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_02/b.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_02/b.yaml deleted file mode 100644 index 05066f22..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_02/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_challenge_alt_yaml -group: arc_challenge_alt_ov_02 -task: arc_challenge_alt_ov_02b -doc_to_text: !function ../styles.template_02 -doc_to_choice: !function ../styles.choice_02b -doc_to_decontamination_query: !function ../styles.template_02 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_02/c.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_02/c.yaml deleted file mode 100644 index ce04fda3..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_02/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_challenge_alt_yaml -group: arc_challenge_alt_ov_02 -task: arc_challenge_alt_ov_02c -doc_to_text: !function ../styles.template_02 -doc_to_choice: !function ../styles.choice_02c -doc_to_decontamination_query: !function ../styles.template_02 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_03/a.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_03/a.yaml deleted file mode 100644 index 864a7912..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_03/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_challenge_alt_yaml -group: arc_challenge_alt_ov_03 -task: arc_challenge_alt_ov_03a -doc_to_text: !function ../styles.template_03 -doc_to_choice: !function ../styles.choice_03a -doc_to_decontamination_query: !function ../styles.template_03 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_03/b.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_03/b.yaml deleted file mode 100644 index c9c2c9d9..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_03/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_challenge_alt_yaml -group: arc_challenge_alt_ov_03 -task: arc_challenge_alt_ov_03b -doc_to_text: !function ../styles.template_03 -doc_to_choice: !function ../styles.choice_03b -doc_to_decontamination_query: !function ../styles.template_03 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_03/c.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_03/c.yaml deleted file mode 100644 index 4e9d0397..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_03/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_challenge_alt_yaml -group: arc_challenge_alt_ov_03 -task: arc_challenge_alt_ov_03c -doc_to_text: !function ../styles.template_03 -doc_to_choice: !function ../styles.choice_03c -doc_to_decontamination_query: !function ../styles.template_03 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_04/a.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_04/a.yaml deleted file mode 100644 index 8061081d..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_04/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_challenge_alt_yaml -group: arc_challenge_alt_ov_04 -task: arc_challenge_alt_ov_04a -doc_to_text: !function ../styles.template_04 -doc_to_choice: !function ../styles.choice_04a -doc_to_decontamination_query: !function ../styles.template_04 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_04/b.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_04/b.yaml deleted file mode 100644 index 91c868a8..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_04/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_challenge_alt_yaml -group: arc_challenge_alt_ov_04 -task: arc_challenge_alt_ov_04b -doc_to_text: !function ../styles.template_04 -doc_to_choice: !function ../styles.choice_04b -doc_to_decontamination_query: !function ../styles.template_04 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_04/c.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_04/c.yaml deleted file mode 100644 index 98581583..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_04/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_challenge_alt_yaml -group: arc_challenge_alt_ov_04 -task: arc_challenge_alt_ov_04c -doc_to_text: !function ../styles.template_04 -doc_to_choice: !function ../styles.choice_04c -doc_to_decontamination_query: !function ../styles.template_04 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_05/a.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_05/a.yaml deleted file mode 100644 index 04fccae4..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_05/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_challenge_alt_yaml -group: arc_challenge_alt_ov_05 -task: arc_challenge_alt_ov_05a -doc_to_text: !function ../styles.template_05 -doc_to_choice: !function ../styles.choice_05a -doc_to_decontamination_query: !function ../styles.template_05 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_05/b.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_05/b.yaml deleted file mode 100644 index eb2a6419..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_05/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_challenge_alt_yaml -group: arc_challenge_alt_ov_05 -task: arc_challenge_alt_ov_05b -doc_to_text: !function ../styles.template_05 -doc_to_choice: !function ../styles.choice_05b -doc_to_decontamination_query: !function ../styles.template_05 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_05/c.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_05/c.yaml deleted file mode 100644 index 757f1c25..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_05/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_challenge_alt_yaml -group: arc_challenge_alt_ov_05 -task: arc_challenge_alt_ov_05c -doc_to_text: !function ../styles.template_05 -doc_to_choice: !function ../styles.choice_05c -doc_to_decontamination_query: !function ../styles.template_05 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_06/a.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_06/a.yaml deleted file mode 100644 index c3f2535e..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_06/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_challenge_alt_yaml -group: arc_challenge_alt_ov_06 -task: arc_challenge_alt_ov_06a -doc_to_text: !function ../styles.template_06 -doc_to_choice: !function ../styles.choice_06a -doc_to_decontamination_query: !function ../styles.template_06 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_06/b.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_06/b.yaml deleted file mode 100644 index d36f1f1a..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_06/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_challenge_alt_yaml -group: arc_challenge_alt_ov_06 -task: arc_challenge_alt_ov_06b -doc_to_text: !function ../styles.template_06 -doc_to_choice: !function ../styles.choice_06b -doc_to_decontamination_query: !function ../styles.template_06 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_06/c.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_06/c.yaml deleted file mode 100644 index fcbfd04c..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_06/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_challenge_alt_yaml -group: arc_challenge_alt_ov_06 -task: arc_challenge_alt_ov_06c -doc_to_text: !function ../styles.template_06 -doc_to_choice: !function ../styles.choice_06c -doc_to_decontamination_query: !function ../styles.template_06 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_07/a.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_07/a.yaml deleted file mode 100644 index 04f6e742..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_07/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_challenge_alt_yaml -group: arc_challenge_alt_ov_07 -task: arc_challenge_alt_ov_07a -doc_to_text: !function ../styles.template_07 -doc_to_choice: !function ../styles.choice_07a -doc_to_decontamination_query: !function ../styles.template_07 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_07/b.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_07/b.yaml deleted file mode 100644 index 1817632a..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_07/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_challenge_alt_yaml -group: arc_challenge_alt_ov_07 -task: arc_challenge_alt_ov_07b -doc_to_text: !function ../styles.template_07 -doc_to_choice: !function ../styles.choice_07b -doc_to_decontamination_query: !function ../styles.template_07 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_07/c.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_07/c.yaml deleted file mode 100644 index 66c376a5..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_07/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_challenge_alt_yaml -group: arc_challenge_alt_ov_07 -task: arc_challenge_alt_ov_07c -doc_to_text: !function ../styles.template_07 -doc_to_choice: !function ../styles.choice_07c -doc_to_decontamination_query: !function ../styles.template_07 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_08/a.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_08/a.yaml deleted file mode 100644 index da9fe33d..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_08/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_challenge_alt_yaml -group: arc_challenge_alt_ov_08 -task: arc_challenge_alt_ov_08a -doc_to_text: !function ../styles.template_08 -doc_to_choice: !function ../styles.choice_08a -doc_to_decontamination_query: !function ../styles.template_08 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_08/b.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_08/b.yaml deleted file mode 100644 index 8fc7c7e8..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_08/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_challenge_alt_yaml -group: arc_challenge_alt_ov_08 -task: arc_challenge_alt_ov_08b -doc_to_text: !function ../styles.template_08 -doc_to_choice: !function ../styles.choice_08b -doc_to_decontamination_query: !function ../styles.template_08 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_08/c.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_08/c.yaml deleted file mode 100644 index 9a12a0b2..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_08/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_challenge_alt_yaml -group: arc_challenge_alt_ov_08 -task: arc_challenge_alt_ov_08c -doc_to_text: !function ../styles.template_08 -doc_to_choice: !function ../styles.choice_08c -doc_to_decontamination_query: !function ../styles.template_08 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/styles.py b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/styles.py deleted file mode 100644 index a4e61348..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/styles.py +++ /dev/null @@ -1,90 +0,0 @@ -import string -from functools import partial - - -def doc_to_text_base(alphabet, style, doc): - - choices = doc["choices"]["text"] - num = len(choices) - - letter_list = [style.format(letter) for letter in alphabet[0:num]] - - if "\t" in style: - choice_string = "{}{}" - else: - choice_string = "{} {}" - - doc_to_text = "\n".join( - [ - "Question: " + doc["question"], - ] - + [choice_string.format(i, j) for i, j in zip(letter_list, choices)] - + ["Answer:"] - ) - - return doc_to_text - - -# Full continuation -def choice_A(doc): - return doc["choices"]["text"] - - -# Letters only -def choice_B(alphabet, style, doc): - - choices = doc["choices"]["text"] - num = len(choices) - - letter_list = [style.format(letter) for letter in alphabet[0:num]] - if "\t" in style: - letter_list = [letter.replace("\t", "") for letter in letter_list] - - return letter_list - - -# Letters + Full continuation -def choice_C(alphabet, style, doc): - - choices = doc["choices"]["text"] - num = len(choices) - - letter_list = [style.format(letter) for letter in alphabet[0:num]] - if "\t" not in style: - letter_list = [letter + " " for letter in letter_list] - - return [letter + choice for letter, choice in zip(letter_list, choices)] - - -template_01 = partial(doc_to_text_base, string.ascii_lowercase, "({})") -choice_01a = choice_A -choice_01b = partial(choice_B, string.ascii_lowercase, "({})") -choice_01c = partial(choice_C, string.ascii_lowercase, "({})") -template_02 = partial(doc_to_text_base, string.ascii_lowercase, "{})") -choice_02a = choice_A -choice_02b = partial(choice_B, string.ascii_lowercase, "{})") -choice_02c = partial(choice_C, string.ascii_lowercase, "{})") -template_03 = partial(doc_to_text_base, string.ascii_lowercase, "{}.") -choice_03a = choice_A -choice_03b = partial(choice_B, string.ascii_lowercase, "{}.") -choice_03c = partial(choice_C, string.ascii_lowercase, "{}.") -template_04 = partial(doc_to_text_base, string.ascii_lowercase, "{}\t") -choice_04a = choice_A -choice_04b = partial(choice_B, string.ascii_lowercase, "{}\t") -choice_04c = partial(choice_C, string.ascii_lowercase, "{}\t") -template_05 = partial(doc_to_text_base, string.ascii_uppercase, "({})") -choice_05a = choice_A -choice_05b = partial(choice_B, string.ascii_uppercase, "({})") -choice_05c = partial(choice_C, string.ascii_uppercase, "({})") -template_06 = partial(doc_to_text_base, string.ascii_uppercase, "{})") -choice_06a = choice_A -choice_06b = partial(choice_B, string.ascii_uppercase, "{})") -choice_06c = partial(choice_C, string.ascii_uppercase, "{})") -template_07 = partial(doc_to_text_base, string.ascii_uppercase, "{}.") -choice_07a = choice_A -choice_07b = partial(choice_B, string.ascii_uppercase, "{}.") -choice_07c = partial(choice_C, string.ascii_uppercase, "{}.") -template_08 = partial(doc_to_text_base, string.ascii_uppercase, "{}\t") -choice_08a = choice_A -choice_08b = partial(choice_B, string.ascii_uppercase, "{}\t") -choice_08c = partial(choice_C, string.ascii_uppercase, "{}\t") diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/prompt_variation/_arc_challenge_alt_yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/prompt_variation/_arc_challenge_alt_yaml deleted file mode 100644 index e8ab9268..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/prompt_variation/_arc_challenge_alt_yaml +++ /dev/null @@ -1,21 +0,0 @@ -dataset_path: ai2_arc -dataset_name: ARC-Challenge -output_type: multiple_choice -training_split: train -validation_split: validation -test_split: test -doc_to_text: "Question: {{question}}\nAnswer:" -doc_to_target: "{{choices.label.index(answerKey)}}" -doc_to_choice: "{{choices.text}}" -should_decontaminate: true -doc_to_decontamination_query: "Question: {{question}}\nAnswer:" -metric_list: - - metric: acc - aggregation: mean - higher_is_better: true - - metric: acc_norm - aggregation: mean - higher_is_better: true - - metric: brier_score - aggregation: brier_score - higher_is_better: false diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/prompt_variation/style_01.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/prompt_variation/style_01.yaml deleted file mode 100644 index e113e7c8..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/prompt_variation/style_01.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: _arc_challenge_alt_yaml -group: arc_challenge_alt_pv -task: arc_challenge_alt_pv_01 -doc_to_text: "{{question}}" -doc_to_decontamination_query: "{{question}}" diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/prompt_variation/style_02.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/prompt_variation/style_02.yaml deleted file mode 100644 index c808c137..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/prompt_variation/style_02.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: _arc_challenge_alt_yaml -group: arc_challenge_alt_pv -task: arc_challenge_alt_pv_02 -doc_to_text: "Q: {{question}}\nA:" -doc_to_decontamination_query: "Q: {{question}}\nA:" diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/prompt_variation/style_03.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_challenge/prompt_variation/style_03.yaml deleted file mode 100644 index 657ed98b..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_challenge/prompt_variation/style_03.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: _arc_challenge_alt_yaml -group: arc_challenge_alt_pv -task: arc_challenge_alt_pv_03 -doc_to_text: "Question: {{question}}\nAnswer:" -doc_to_decontamination_query: "Question: {{question}}\nAnswer:" diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/README.md b/lm_eval/tasks/arc/alternative_worlds/arc_easy/README.md deleted file mode 100644 index 93600ae1..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/README.md +++ /dev/null @@ -1,20 +0,0 @@ - - -Investigate affect of letter options -- (A) -- A) -- A. -- A\t -- (a) -- a) -- a. -- a\t - -Answer types: -- letters only - - original option - - just letter -- letters + continuation - - original option - - just letter -- continuation diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/_arc_easy_alt_yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/_arc_easy_alt_yaml deleted file mode 100644 index 633826a2..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/_arc_easy_alt_yaml +++ /dev/null @@ -1,24 +0,0 @@ -group: - - ai2_arc -task: arc_easy -dataset_path: ai2_arc -dataset_name: ARC-Easy -output_type: multiple_choice -training_split: train -validation_split: validation -test_split: test -doc_to_text: "Question: {{question}}\nAnswer:" -doc_to_target: "{{choices.label.index(answerKey)}}" -doc_to_choice: "{{choices.text}}" -should_decontaminate: true -doc_to_decontamination_query: "Question: {{question}}\nAnswer:" -metric_list: - - metric: acc - aggregation: mean - higher_is_better: true - - metric: acc_norm - aggregation: mean - higher_is_better: true - - metric: brier_score - aggregation: brier_score - higher_is_better: false diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/arc_easy_alt.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/arc_easy_alt.yaml deleted file mode 100644 index af4dcf97..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/arc_easy_alt.yaml +++ /dev/null @@ -1,10 +0,0 @@ -group: arc_easy_alt_ov -task: - - arc_easy_alt_ov_01 - - arc_easy_alt_ov_02 - - arc_easy_alt_ov_03 - - arc_easy_alt_ov_04 - - arc_easy_alt_ov_05 - - arc_easy_alt_ov_06 - - arc_easy_alt_ov_07 - - arc_easy_alt_ov_08 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_01/a.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_01/a.yaml deleted file mode 100644 index 784262d7..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_01/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_easy_alt_yaml -group: arc_easy_alt_ov_01 -task: arc_easy_alt_ov_01a -doc_to_text: !function ../styles.template_01 -doc_to_choice: !function ../styles.choice_01a -doc_to_decontamination_query: !function ../styles.template_01 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_01/b.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_01/b.yaml deleted file mode 100644 index 29958b6c..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_01/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_easy_alt_yaml -group: arc_easy_alt_ov_01 -task: arc_easy_alt_ov_01b -doc_to_text: !function ../styles.template_01 -doc_to_choice: !function ../styles.choice_01b -doc_to_decontamination_query: !function ../styles.template_01 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_01/c.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_01/c.yaml deleted file mode 100644 index a2bcae1a..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_01/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_easy_alt_yaml -group: arc_easy_alt_ov_01 -task: arc_easy_alt_ov_01c -doc_to_text: !function ../styles.template_01 -doc_to_choice: !function ../styles.choice_01c -doc_to_decontamination_query: !function ../styles.template_01 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_02/a.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_02/a.yaml deleted file mode 100644 index 6cb6d701..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_02/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_easy_alt_yaml -group: arc_easy_alt_ov_02 -task: arc_easy_alt_ov_02a -doc_to_text: !function ../styles.template_02 -doc_to_choice: !function ../styles.choice_02a -doc_to_decontamination_query: !function ../styles.template_02 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_02/b.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_02/b.yaml deleted file mode 100644 index b8afda44..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_02/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_easy_alt_yaml -group: arc_easy_alt_ov_02 -task: arc_easy_alt_ov_02b -doc_to_text: !function ../styles.template_02 -doc_to_choice: !function ../styles.choice_02b -doc_to_decontamination_query: !function ../styles.template_02 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_02/c.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_02/c.yaml deleted file mode 100644 index a9a8b71a..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_02/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_easy_alt_yaml -group: arc_easy_alt_ov_02 -task: arc_easy_alt_ov_02c -doc_to_text: !function ../styles.template_02 -doc_to_choice: !function ../styles.choice_02c -doc_to_decontamination_query: !function ../styles.template_02 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_03/a.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_03/a.yaml deleted file mode 100644 index a553e313..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_03/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_easy_alt_yaml -group: arc_easy_alt_ov_03 -task: arc_easy_alt_ov_03a -doc_to_text: !function ../styles.template_03 -doc_to_choice: !function ../styles.choice_03a -doc_to_decontamination_query: !function ../styles.template_03 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_03/b.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_03/b.yaml deleted file mode 100644 index fc7e4339..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_03/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_easy_alt_yaml -group: arc_easy_alt_ov_03 -task: arc_easy_alt_ov_03b -doc_to_text: !function ../styles.template_03 -doc_to_choice: !function ../styles.choice_03b -doc_to_decontamination_query: !function ../styles.template_03 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_03/c.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_03/c.yaml deleted file mode 100644 index bd80d2db..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_03/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_easy_alt_yaml -group: arc_easy_alt_ov_03 -task: arc_easy_alt_ov_03c -doc_to_text: !function ../styles.template_03 -doc_to_choice: !function ../styles.choice_03c -doc_to_decontamination_query: !function ../styles.template_03 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_04/a.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_04/a.yaml deleted file mode 100644 index 307809e8..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_04/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_easy_alt_yaml -group: arc_easy_alt_ov_04 -task: arc_easy_alt_ov_04a -doc_to_text: !function ../styles.template_04 -doc_to_choice: !function ../styles.choice_04a -doc_to_decontamination_query: !function ../styles.template_04 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_04/b.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_04/b.yaml deleted file mode 100644 index 81c88c45..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_04/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_easy_alt_yaml -group: arc_easy_alt_ov_04 -task: arc_easy_alt_ov_04b -doc_to_text: !function ../styles.template_04 -doc_to_choice: !function ../styles.choice_04b -doc_to_decontamination_query: !function ../styles.template_04 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_04/c.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_04/c.yaml deleted file mode 100644 index 3c18665d..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_04/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_easy_alt_yaml -group: arc_easy_alt_ov_04 -task: arc_easy_alt_ov_04c -doc_to_text: !function ../styles.template_04 -doc_to_choice: !function ../styles.choice_04c -doc_to_decontamination_query: !function ../styles.template_04 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_05/a.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_05/a.yaml deleted file mode 100644 index d4393910..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_05/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_easy_alt_yaml -group: arc_easy_alt_ov_05 -task: arc_easy_alt_ov_05a -doc_to_text: !function ../styles.template_05 -doc_to_choice: !function ../styles.choice_05a -doc_to_decontamination_query: !function ../styles.template_05 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_05/b.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_05/b.yaml deleted file mode 100644 index 9fa728a1..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_05/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_easy_alt_yaml -group: arc_easy_alt_ov_05 -task: arc_easy_alt_ov_05b -doc_to_text: !function ../styles.template_05 -doc_to_choice: !function ../styles.choice_05b -doc_to_decontamination_query: !function ../styles.template_05 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_05/c.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_05/c.yaml deleted file mode 100644 index eb8c3f71..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_05/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_easy_alt_yaml -group: arc_easy_alt_ov_05 -task: arc_easy_alt_ov_05c -doc_to_text: !function ../styles.template_05 -doc_to_choice: !function ../styles.choice_05c -doc_to_decontamination_query: !function ../styles.template_05 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_06/a.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_06/a.yaml deleted file mode 100644 index 47141d57..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_06/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_easy_alt_yaml -group: arc_easy_alt_ov_06 -task: arc_easy_alt_ov_06a -doc_to_text: !function ../styles.template_06 -doc_to_choice: !function ../styles.choice_06a -doc_to_decontamination_query: !function ../styles.template_06 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_06/b.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_06/b.yaml deleted file mode 100644 index 2ba20727..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_06/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_easy_alt_yaml -group: arc_easy_alt_ov_06 -task: arc_easy_alt_ov_06b -doc_to_text: !function ../styles.template_06 -doc_to_choice: !function ../styles.choice_06b -doc_to_decontamination_query: !function ../styles.template_06 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_06/c.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_06/c.yaml deleted file mode 100644 index 04c7fc4d..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_06/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_easy_alt_yaml -group: arc_easy_alt_ov_06 -task: arc_easy_alt_ov_06c -doc_to_text: !function ../styles.template_06 -doc_to_choice: !function ../styles.choice_06c -doc_to_decontamination_query: !function ../styles.template_06 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_07/a.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_07/a.yaml deleted file mode 100644 index 8bd4168c..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_07/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_easy_alt_yaml -group: arc_easy_alt_ov_07 -task: arc_easy_alt_ov_07a -doc_to_text: !function ../styles.template_07 -doc_to_choice: !function ../styles.choice_07a -doc_to_decontamination_query: !function ../styles.template_07 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_07/b.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_07/b.yaml deleted file mode 100644 index 6e797f23..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_07/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_easy_alt_yaml -group: arc_easy_alt_ov_07 -task: arc_easy_alt_ov_07b -doc_to_text: !function ../styles.template_07 -doc_to_choice: !function ../styles.choice_07b -doc_to_decontamination_query: !function ../styles.template_07 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_07/c.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_07/c.yaml deleted file mode 100644 index 145098cd..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_07/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_easy_alt_yaml -group: arc_easy_alt_ov_07 -task: arc_easy_alt_ov_07c -doc_to_text: !function ../styles.template_07 -doc_to_choice: !function ../styles.choice_07c -doc_to_decontamination_query: !function ../styles.template_07 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_08/a.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_08/a.yaml deleted file mode 100644 index 84f41d12..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_08/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_easy_alt_yaml -group: arc_easy_alt_ov_08 -task: arc_easy_alt_ov_08a -doc_to_text: !function ../styles.template_08 -doc_to_choice: !function ../styles.choice_08a -doc_to_decontamination_query: !function ../styles.template_08 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_08/b.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_08/b.yaml deleted file mode 100644 index 5e30c8ac..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_08/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_easy_alt_yaml -group: arc_easy_alt_ov_08 -task: arc_easy_alt_ov_08b -doc_to_text: !function ../styles.template_08 -doc_to_choice: !function ../styles.choice_08b -doc_to_decontamination_query: !function ../styles.template_08 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_08/c.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_08/c.yaml deleted file mode 100644 index 4c829aa8..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/style_08/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_arc_easy_alt_yaml -group: arc_easy_alt_ov_08 -task: arc_easy_alt_ov_08c -doc_to_text: !function ../styles.template_08 -doc_to_choice: !function ../styles.choice_08c -doc_to_decontamination_query: !function ../styles.template_08 diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/styles.py b/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/styles.py deleted file mode 100644 index a4e61348..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/output_variation/styles.py +++ /dev/null @@ -1,90 +0,0 @@ -import string -from functools import partial - - -def doc_to_text_base(alphabet, style, doc): - - choices = doc["choices"]["text"] - num = len(choices) - - letter_list = [style.format(letter) for letter in alphabet[0:num]] - - if "\t" in style: - choice_string = "{}{}" - else: - choice_string = "{} {}" - - doc_to_text = "\n".join( - [ - "Question: " + doc["question"], - ] - + [choice_string.format(i, j) for i, j in zip(letter_list, choices)] - + ["Answer:"] - ) - - return doc_to_text - - -# Full continuation -def choice_A(doc): - return doc["choices"]["text"] - - -# Letters only -def choice_B(alphabet, style, doc): - - choices = doc["choices"]["text"] - num = len(choices) - - letter_list = [style.format(letter) for letter in alphabet[0:num]] - if "\t" in style: - letter_list = [letter.replace("\t", "") for letter in letter_list] - - return letter_list - - -# Letters + Full continuation -def choice_C(alphabet, style, doc): - - choices = doc["choices"]["text"] - num = len(choices) - - letter_list = [style.format(letter) for letter in alphabet[0:num]] - if "\t" not in style: - letter_list = [letter + " " for letter in letter_list] - - return [letter + choice for letter, choice in zip(letter_list, choices)] - - -template_01 = partial(doc_to_text_base, string.ascii_lowercase, "({})") -choice_01a = choice_A -choice_01b = partial(choice_B, string.ascii_lowercase, "({})") -choice_01c = partial(choice_C, string.ascii_lowercase, "({})") -template_02 = partial(doc_to_text_base, string.ascii_lowercase, "{})") -choice_02a = choice_A -choice_02b = partial(choice_B, string.ascii_lowercase, "{})") -choice_02c = partial(choice_C, string.ascii_lowercase, "{})") -template_03 = partial(doc_to_text_base, string.ascii_lowercase, "{}.") -choice_03a = choice_A -choice_03b = partial(choice_B, string.ascii_lowercase, "{}.") -choice_03c = partial(choice_C, string.ascii_lowercase, "{}.") -template_04 = partial(doc_to_text_base, string.ascii_lowercase, "{}\t") -choice_04a = choice_A -choice_04b = partial(choice_B, string.ascii_lowercase, "{}\t") -choice_04c = partial(choice_C, string.ascii_lowercase, "{}\t") -template_05 = partial(doc_to_text_base, string.ascii_uppercase, "({})") -choice_05a = choice_A -choice_05b = partial(choice_B, string.ascii_uppercase, "({})") -choice_05c = partial(choice_C, string.ascii_uppercase, "({})") -template_06 = partial(doc_to_text_base, string.ascii_uppercase, "{})") -choice_06a = choice_A -choice_06b = partial(choice_B, string.ascii_uppercase, "{})") -choice_06c = partial(choice_C, string.ascii_uppercase, "{})") -template_07 = partial(doc_to_text_base, string.ascii_uppercase, "{}.") -choice_07a = choice_A -choice_07b = partial(choice_B, string.ascii_uppercase, "{}.") -choice_07c = partial(choice_C, string.ascii_uppercase, "{}.") -template_08 = partial(doc_to_text_base, string.ascii_uppercase, "{}\t") -choice_08a = choice_A -choice_08b = partial(choice_B, string.ascii_uppercase, "{}\t") -choice_08c = partial(choice_C, string.ascii_uppercase, "{}\t") diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/prompt_variation/_arc_easy_alt_yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/prompt_variation/_arc_easy_alt_yaml deleted file mode 100644 index b17b217e..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/prompt_variation/_arc_easy_alt_yaml +++ /dev/null @@ -1,21 +0,0 @@ -dataset_path: ai2_arc -dataset_name: ARC-Easy -output_type: multiple_choice -training_split: train -validation_split: validation -test_split: test -doc_to_text: "Question: {{question}}\nAnswer:" -doc_to_target: "{{choices.label.index(answerKey)}}" -doc_to_choice: "{{choices.text}}" -should_decontaminate: true -doc_to_decontamination_query: "Question: {{question}}\nAnswer:" -metric_list: - - metric: acc - aggregation: mean - higher_is_better: true - - metric: acc_norm - aggregation: mean - higher_is_better: true - - metric: brier_score - aggregation: brier_score - higher_is_better: false diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/prompt_variation/style_01.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/prompt_variation/style_01.yaml deleted file mode 100644 index 614f3970..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/prompt_variation/style_01.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: _arc_easy_alt_yaml -group: arc_easy_alt_pv -task: arc_easy_alt_pv_01 -doc_to_text: "{{question}}" -doc_to_decontamination_query: "{{question}}" diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/prompt_variation/style_02.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/prompt_variation/style_02.yaml deleted file mode 100644 index 3bc2e7e9..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/prompt_variation/style_02.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: _arc_easy_alt_yaml -group: arc_easy_alt_pv -task: arc_easy_alt_pv_02 -doc_to_text: "Q: {{question}}\nA:" -doc_to_decontamination_query: "Q: {{question}}\nA:" diff --git a/lm_eval/tasks/arc/alternative_worlds/arc_easy/prompt_variation/style_03.yaml b/lm_eval/tasks/arc/alternative_worlds/arc_easy/prompt_variation/style_03.yaml deleted file mode 100644 index a108887f..00000000 --- a/lm_eval/tasks/arc/alternative_worlds/arc_easy/prompt_variation/style_03.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: _arc_easy_alt_yaml -group: arc_easy_alt_pv -task: arc_easy_alt_pv_03 -doc_to_text: "Question: {{question}}\nAnswer:" -doc_to_decontamination_query: "Question: {{question}}\nAnswer:" diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/_template_yaml b/lm_eval/tasks/arithmetic/alternative_worlds/_template_yaml deleted file mode 100644 index b68001e1..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/_template_yaml +++ /dev/null @@ -1,13 +0,0 @@ -dataset_path: EleutherAI/arithmetic -dataset_name: arithmetic_1dc -output_type: loglikelihood -validation_split: validation -test_split: null -doc_to_text: "{{context}}" -doc_to_target: "{{completion}}" -metric_list: - - metric: acc - aggregation: mean - higher_is_better: true - - metric: brier_score - higher_is_better: false diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/arithmetic_alt.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/arithmetic_alt.yaml deleted file mode 100644 index 1e678c04..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/arithmetic_alt.yaml +++ /dev/null @@ -1,8 +0,0 @@ -group: arithmetic_alt -task: - - arithmetic_alt_00 - - arithmetic_alt_01 - - arithmetic_alt_02 - - arithmetic_alt_03 - - arithmetic_alt_04 - - arithmetic_alt_05 diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_00/_template_00_yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_00/_template_00_yaml deleted file mode 100644 index 43b73a88..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_00/_template_00_yaml +++ /dev/null @@ -1,15 +0,0 @@ -include: ../_template_yaml -group: arithmetic_alt_00 -group_alias: arithmetic (Style 00) -dataset_path: EleutherAI/arithmetic -output_type: loglikelihood -validation_split: validation -test_split: null -doc_to_text: !function ../utils.style_00 -doc_to_target: "{{completion}}" -metric_list: - - metric: acc - aggregation: mean - higher_is_better: true - - metric: brier_score - higher_is_better: false diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_1dc.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_1dc.yaml deleted file mode 100644 index 0c989c6f..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_1dc.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_00_yaml -task: arithmetic_1dc_alt_00 -dataset_name: arithmetic_1dc -task_alias: 1dc diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_2da.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_2da.yaml deleted file mode 100644 index 11501c8f..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_2da.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_00_yaml -task: arithmetic_2da_alt_00 -dataset_name: arithmetic_2da -task_alias: 2da diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_2dm.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_2dm.yaml deleted file mode 100644 index 7a0d35b1..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_2dm.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_00_yaml -task: arithmetic_2dm_alt_00 -dataset_name: arithmetic_2dm -task_alias: 2dm diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_2ds.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_2ds.yaml deleted file mode 100644 index 0720c833..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_2ds.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_00_yaml -task: arithmetic_2ds_alt_00 -dataset_name: arithmetic_2ds -task_alias: 2ds diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_3da.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_3da.yaml deleted file mode 100644 index 3f6cf88a..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_3da.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_00_yaml -task: arithmetic_3da_alt_00 -dataset_name: arithmetic_3da -task_alias: 3da diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_3ds.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_3ds.yaml deleted file mode 100644 index a83dffe0..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_3ds.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_00_yaml -task: arithmetic_3ds_alt_00 -dataset_name: arithmetic_3ds -task_alias: 3ds diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_4da.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_4da.yaml deleted file mode 100644 index 80999856..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_4da.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_00_yaml -task: arithmetic_4da_alt_00 -dataset_name: arithmetic_4da -task_alias: 4da diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_4ds.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_4ds.yaml deleted file mode 100644 index 59a6e5a0..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_4ds.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_00_yaml -task: arithmetic_4ds_alt_00 -dataset_name: arithmetic_4ds -task_alias: 4ds diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_5da.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_5da.yaml deleted file mode 100644 index f172341b..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_5da.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_00_yaml -task: arithmetic_5da_alt_00 -dataset_name: arithmetic_5da -task_alias: 5da diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_5ds.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_5ds.yaml deleted file mode 100644 index 2da65545..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_5ds.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_00_yaml -task: arithmetic_5ds_alt_00 -dataset_name: arithmetic_5ds -task_alias: 5ds diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_01/_template_01_yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_01/_template_01_yaml deleted file mode 100644 index 5f10c542..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_01/_template_01_yaml +++ /dev/null @@ -1,15 +0,0 @@ -include: ../_template_yaml -group: arithmetic_alt_01 -group_alias: arithmetic (Style 01) -dataset_path: EleutherAI/arithmetic -output_type: loglikelihood -validation_split: validation -test_split: null -doc_to_text: !function ../utils.style_01 -doc_to_target: "{{completion}}" -metric_list: - - metric: acc - aggregation: mean - higher_is_better: true - - metric: brier_score - higher_is_better: false diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_1dc.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_1dc.yaml deleted file mode 100644 index 2764ccb2..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_1dc.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_01_yaml -task: arithmetic_1dc_alt_01 -dataset_name: arithmetic_1dc -task_alias: 1dc diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_2da.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_2da.yaml deleted file mode 100644 index 753eef6a..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_2da.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_01_yaml -task: arithmetic_2da_alt_01 -dataset_name: arithmetic_2da -task_alias: 2da diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_2dm.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_2dm.yaml deleted file mode 100644 index abbbecc4..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_2dm.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_01_yaml -task: arithmetic_2dm_alt_01 -dataset_name: arithmetic_2dm -task_alias: 2dm diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_2ds.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_2ds.yaml deleted file mode 100644 index 3f069e28..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_2ds.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_01_yaml -task: arithmetic_2ds_alt_01 -dataset_name: arithmetic_2ds -task_alias: 2ds diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_3da.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_3da.yaml deleted file mode 100644 index 6fa481f5..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_3da.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_01_yaml -task: arithmetic_3da_alt_01 -dataset_name: arithmetic_3da -task_alias: 3da diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_3ds.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_3ds.yaml deleted file mode 100644 index 5e8d27b4..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_3ds.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_01_yaml -task: arithmetic_3ds_alt_01 -dataset_name: arithmetic_3ds -task_alias: 3ds diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_4da.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_4da.yaml deleted file mode 100644 index 5564a98c..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_4da.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_01_yaml -task: arithmetic_4da_alt_01 -dataset_name: arithmetic_4da -task_alias: 4da diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_4ds.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_4ds.yaml deleted file mode 100644 index 765400b6..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_4ds.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_01_yaml -task: arithmetic_4ds_alt_01 -dataset_name: arithmetic_4ds -task_alias: 4ds diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_5da.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_5da.yaml deleted file mode 100644 index 44548858..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_5da.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_01_yaml -task: arithmetic_5da_alt_01 -dataset_name: arithmetic_5da -task_alias: 5da diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_5ds.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_5ds.yaml deleted file mode 100644 index 571bc80d..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_01/arithmetic_5ds.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_01_yaml -task: arithmetic_5ds_alt_01 -dataset_name: arithmetic_5ds -task_alias: 5ds diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_02/_template_02_yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_02/_template_02_yaml deleted file mode 100644 index 2e2a3f5c..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_02/_template_02_yaml +++ /dev/null @@ -1,15 +0,0 @@ -include: ../_template_yaml -group: arithmetic_alt_02 -group_alias: arithmetic (Style 02) -dataset_path: EleutherAI/arithmetic -output_type: loglikelihood -validation_split: validation -test_split: null -doc_to_text: !function ../utils.style_02 -doc_to_target: "{{completion}}" -metric_list: - - metric: acc - aggregation: mean - higher_is_better: true - - metric: brier_score - higher_is_better: false diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_1dc.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_1dc.yaml deleted file mode 100644 index 6d3184da..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_1dc.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_02_yaml -task: arithmetic_1dc_alt_02 -dataset_name: arithmetic_1dc -task_alias: 1dc diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_2da.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_2da.yaml deleted file mode 100644 index 2a7a9e59..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_2da.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_02_yaml -task: arithmetic_2da_alt_02 -dataset_name: arithmetic_2da -task_alias: 2da diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_2dm.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_2dm.yaml deleted file mode 100644 index 30c38a6d..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_2dm.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_02_yaml -task: arithmetic_2dm_alt_02 -dataset_name: arithmetic_2dm -task_alias: 2dm diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_2ds.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_2ds.yaml deleted file mode 100644 index ab18bce0..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_2ds.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_02_yaml -task: arithmetic_2ds_alt_02 -dataset_name: arithmetic_2ds -task_alias: 2ds diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_3da.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_3da.yaml deleted file mode 100644 index e59c3798..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_3da.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_02_yaml -task: arithmetic_3da_alt_02 -dataset_name: arithmetic_3da -task_alias: 3da diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_3ds.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_3ds.yaml deleted file mode 100644 index 6d37ef2a..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_3ds.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_02_yaml -task: arithmetic_3ds_alt_02 -dataset_name: arithmetic_3ds -task_alias: 3ds diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_4da.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_4da.yaml deleted file mode 100644 index eabbb1a0..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_4da.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_02_yaml -task: arithmetic_4da_alt_02 -dataset_name: arithmetic_4da -task_alias: 4da diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_4ds.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_4ds.yaml deleted file mode 100644 index 8ce2bb79..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_4ds.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_02_yaml -task: arithmetic_4ds_alt_02 -dataset_name: arithmetic_4ds -task_alias: 4ds diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_5da.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_5da.yaml deleted file mode 100644 index 48ef1aaa..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_5da.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_02_yaml -task: arithmetic_5da_alt_02 -dataset_name: arithmetic_5da -task_alias: 5da diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_5ds.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_5ds.yaml deleted file mode 100644 index 1941ff65..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_02/arithmetic_5ds.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_02_yaml -task: arithmetic_5ds_alt_02 -dataset_name: arithmetic_5ds -task_alias: 5ds diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_03/_template_03_yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_03/_template_03_yaml deleted file mode 100644 index 275559d1..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_03/_template_03_yaml +++ /dev/null @@ -1,15 +0,0 @@ -include: ../_template_yaml -group: arithmetic_alt_03 -group_alias: arithmetic (Style 03) -dataset_path: EleutherAI/arithmetic -output_type: loglikelihood -validation_split: validation -test_split: null -doc_to_text: !function ../utils.style_03 -doc_to_target: "{{completion}}" -metric_list: - - metric: acc - aggregation: mean - higher_is_better: true - - metric: brier_score - higher_is_better: false diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_1dc.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_1dc.yaml deleted file mode 100644 index c68279f4..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_1dc.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_03_yaml -task: arithmetic_1dc_alt_03 -dataset_name: arithmetic_1dc -task_alias: 1dc diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_2da.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_2da.yaml deleted file mode 100644 index 0ef30ee7..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_2da.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_03_yaml -task: arithmetic_2da_alt_03 -dataset_name: arithmetic_2da -task_alias: 2da diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_2dm.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_2dm.yaml deleted file mode 100644 index f742d37b..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_2dm.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_03_yaml -task: arithmetic_2dm_alt_03 -dataset_name: arithmetic_2dm -task_alias: 2dm diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_2ds.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_2ds.yaml deleted file mode 100644 index 59a85bfb..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_2ds.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_03_yaml -task: arithmetic_2ds_alt_03 -dataset_name: arithmetic_2ds -task_alias: 2ds diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_3da.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_3da.yaml deleted file mode 100644 index d4e4d772..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_3da.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_03_yaml -task: arithmetic_3da_alt_03 -dataset_name: arithmetic_3da -task_alias: 3da diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_3ds.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_3ds.yaml deleted file mode 100644 index 7022a855..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_3ds.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_03_yaml -task: arithmetic_3ds_alt_03 -dataset_name: arithmetic_3ds -task_alias: 3ds diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_4da.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_4da.yaml deleted file mode 100644 index 0d70f481..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_4da.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_03_yaml -task: arithmetic_4da_alt_03 -dataset_name: arithmetic_4da -task_alias: 4da diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_4ds.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_4ds.yaml deleted file mode 100644 index 77cd5c65..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_4ds.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_03_yaml -task: arithmetic_4ds_alt_03 -dataset_name: arithmetic_4ds -task_alias: 4ds diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_5da.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_5da.yaml deleted file mode 100644 index 02aec1d1..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_5da.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_03_yaml -task: arithmetic_5da_alt_03 -dataset_name: arithmetic_5da -task_alias: 5da diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_5ds.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_5ds.yaml deleted file mode 100644 index 6bbbd3a1..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_03/arithmetic_5ds.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_03_yaml -task: arithmetic_5ds_alt_03 -dataset_name: arithmetic_5ds -task_alias: 5ds diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_04/_template_04_yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_04/_template_04_yaml deleted file mode 100644 index 6680afad..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_04/_template_04_yaml +++ /dev/null @@ -1,15 +0,0 @@ -include: ../_template_yaml -group: arithmetic_alt_04 -group_alias: arithmetic (Style 04) -dataset_path: EleutherAI/arithmetic -output_type: loglikelihood -validation_split: validation -test_split: null -doc_to_text: !function ../utils.style_04 -doc_to_target: "{{completion}}" -metric_list: - - metric: acc - aggregation: mean - higher_is_better: true - - metric: brier_score - higher_is_better: false diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_1dc.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_1dc.yaml deleted file mode 100644 index 8d940e1c..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_1dc.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_04_yaml -task: arithmetic_1dc_alt_04 -dataset_name: arithmetic_1dc -task_alias: 1dc diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_2da.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_2da.yaml deleted file mode 100644 index 0627fe5e..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_2da.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_04_yaml -task: arithmetic_2da_alt_04 -dataset_name: arithmetic_2da -task_alias: 2da diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_2dm.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_2dm.yaml deleted file mode 100644 index 667259c8..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_2dm.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_04_yaml -task: arithmetic_2dm_alt_04 -dataset_name: arithmetic_2dm -task_alias: 2dm diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_2ds.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_2ds.yaml deleted file mode 100644 index fc45ee63..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_2ds.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_04_yaml -task: arithmetic_2ds_alt_04 -dataset_name: arithmetic_2ds -task_alias: 2ds diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_3da.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_3da.yaml deleted file mode 100644 index d1468745..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_3da.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_04_yaml -task: arithmetic_3da_alt_04 -dataset_name: arithmetic_3da -task_alias: 3da diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_3ds.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_3ds.yaml deleted file mode 100644 index d67daf13..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_3ds.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_04_yaml -task: arithmetic_3ds_alt_04 -dataset_name: arithmetic_3ds -task_alias: 3ds diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_4da.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_4da.yaml deleted file mode 100644 index 339c47f4..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_4da.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_04_yaml -task: arithmetic_4da_alt_04 -dataset_name: arithmetic_4da -task_alias: 4da diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_4ds.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_4ds.yaml deleted file mode 100644 index c983c896..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_4ds.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_04_yaml -task: arithmetic_4ds_alt_04 -dataset_name: arithmetic_4ds -task_alias: 4ds diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_5da.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_5da.yaml deleted file mode 100644 index 7e9cbf0f..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_5da.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_04_yaml -task: arithmetic_5da_alt_04 -dataset_name: arithmetic_5da -task_alias: 5da diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_5ds.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_5ds.yaml deleted file mode 100644 index dca4bf98..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_04/arithmetic_5ds.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_04_yaml -task: arithmetic_5ds_alt_04 -dataset_name: arithmetic_5ds -task_alias: 5ds diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_05/_template_05_yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_05/_template_05_yaml deleted file mode 100644 index f838a10e..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_05/_template_05_yaml +++ /dev/null @@ -1,15 +0,0 @@ -include: ../_template_yaml -group: arithmetic_alt_05 -group_alias: arithmetic (Style 05) -dataset_path: EleutherAI/arithmetic -output_type: loglikelihood -validation_split: validation -test_split: null -doc_to_text: !function ../utils.style_05 -doc_to_target: "{{completion}}" -metric_list: - - metric: acc - aggregation: mean - higher_is_better: true - - metric: brier_score - higher_is_better: false diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_1dc.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_1dc.yaml deleted file mode 100644 index 2c0f4cec..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_1dc.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_05_yaml -task: arithmetic_1dc_alt_05 -dataset_name: arithmetic_1dc -task_alias: 1dc diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_2da.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_2da.yaml deleted file mode 100644 index 94632d23..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_2da.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_05_yaml -task: arithmetic_2da_alt_05 -dataset_name: arithmetic_2da -task_alias: 2da diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_2dm.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_2dm.yaml deleted file mode 100644 index df44aa68..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_2dm.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_05_yaml -task: arithmetic_2dm_alt_05 -dataset_name: arithmetic_2dm -task_alias: 2dm diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_2ds.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_2ds.yaml deleted file mode 100644 index d34f87d1..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_2ds.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_05_yaml -task: arithmetic_2ds_alt_05 -dataset_name: arithmetic_2ds -task_alias: 2ds diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_3da.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_3da.yaml deleted file mode 100644 index 5bbd6b9e..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_3da.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_05_yaml -task: arithmetic_3da_alt_05 -dataset_name: arithmetic_3da -task_alias: 3da diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_3ds.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_3ds.yaml deleted file mode 100644 index 32d15f72..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_3ds.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_05_yaml -task: arithmetic_3ds_alt_05 -dataset_name: arithmetic_3ds -task_alias: 3ds diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_4da.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_4da.yaml deleted file mode 100644 index 09b7fda5..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_4da.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_05_yaml -task: arithmetic_4da_alt_05 -dataset_name: arithmetic_4da -task_alias: 4da diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_4ds.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_4ds.yaml deleted file mode 100644 index 2d4e4c49..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_4ds.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_05_yaml -task: arithmetic_4ds_alt_05 -dataset_name: arithmetic_4ds -task_alias: 4ds diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_5da.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_5da.yaml deleted file mode 100644 index 12b88bc3..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_5da.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_05_yaml -task: arithmetic_5da_alt_05 -dataset_name: arithmetic_5da -task_alias: 5da diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_5ds.yaml b/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_5ds.yaml deleted file mode 100644 index 406926a2..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/style_05/arithmetic_5ds.yaml +++ /dev/null @@ -1,4 +0,0 @@ -include: _template_05_yaml -task: arithmetic_5ds_alt_05 -dataset_name: arithmetic_5ds -task_alias: 5ds diff --git a/lm_eval/tasks/arithmetic/alternative_worlds/utils.py b/lm_eval/tasks/arithmetic/alternative_worlds/utils.py deleted file mode 100644 index 56ad74ac..00000000 --- a/lm_eval/tasks/arithmetic/alternative_worlds/utils.py +++ /dev/null @@ -1,36 +0,0 @@ -import re - -# Original Prompt -# Question: What is (9 + 8) * 2? Answer: - - -def style_00(docs): - # What is (9 + 8) * 2? - return docs["context"] - - -def style_01(docs): - # What is (9 + 8) * 2? - return docs["context"].replace("Question: ", "").replace(" Answer:", "") - - -def style_02(docs): - # Q: What is (9 + 8) * 2? A: - return docs["context"].replace("Question: ", "Q: ").replace(" Answer:", " A:") - - -def style_03(docs): - # Solve (9 + 8) * 2. - return ( - docs["context"].replace("Question: What is", "Solve").replace(" Answer:", ".") - ) - - -def style_04(docs): - # (9 + 8) * 2 = - return docs["context"].replace("Question: What is ", "").replace(" Answer:", " =") - - -def style_05(docs): - # What is (9 + 8) * 2? Answer: - return docs["context"].replace("Question: ", "") diff --git a/lm_eval/tasks/bbh/alternative_worlds/README.md b/lm_eval/tasks/bbh/alternative_worlds/README.md deleted file mode 100644 index b1412fa3..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/README.md +++ /dev/null @@ -1,34 +0,0 @@ -| Task | Prompt Variation | Output Variation | Option in Sample | -| :-----------------:| :---------------: | :---------------: |:---------------: | -| boolean_expression | Yes | Yes | No | -| causal_judgement | Yes | Yes | Yes | -| date_understanding | Yes | Yes | Yes | -| disambiguation_qa | Yes | Yes | Yes | -| dyck_languages | Yes | No | No | -| formal_fallacies | Yes | Yes | Yes | -| geometric_shapes | Yes | Yes | Yes | -| hyperbaton | Yes | Yes | Yes | -| logical_deduction_five_objects| Yes | Yes | Yes | -| logical_deduction_seven_objects| Yes | Yes | Yes | -| logical_deduction_three_objects| Yes | Yes | Yes | -| movie_recommendation| Yes | Yes | Yes | -| multistep_arithmetic_two| Yes | No | No | -| navigate | Yes | Yes | Yes | -| object_counting | Yes | No | No | -| penguins_in_a_table| Yes | Yes | Yes | -| reasoning_about_colored_objects| Yes | Yes | Yes | -| ruin_names | Yes | Yes | Yes | -| salient_translation_error_detection| Yes| Yes | Yes | -| snarks | Yes | Yes | Yes | -| sports_understanding| Yes | Yes | No | -| temporal_sequences | Yes | Yes | Yes | -| tracking_shuffled_objects_five_objects| Yes| Yes | Yes | -| tracking_shuffled_objects_seven_objects| Yes| Yes | Yes | -| tracking_shuffled_objects_three_objects| Yes| Yes | Yes | -| web_of_lies | Yes | Yes | No | -| word_sorting | Yes | No | No | - - -Notes: -- `web_of_lies` already starts with `Question: ` -- Tasks with options are `Options: (A) ...` (multiple choice) or `Options: - ...` (binary choice) diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/bbh_alt_pv_zeroshot.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/bbh_alt_pv_zeroshot.yaml deleted file mode 100644 index 023037f6..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/bbh_alt_pv_zeroshot.yaml +++ /dev/null @@ -1,5 +0,0 @@ -group: bbh_alt_pv_zeroshot -task: - - bbh_alt_pv_01_zeroshot - - bbh_alt_pv_02_zeroshot - - bbh_alt_pv_03_zeroshot diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/_zeroshot_template_yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/_zeroshot_template_yaml deleted file mode 100644 index 817b2c27..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/_zeroshot_template_yaml +++ /dev/null @@ -1,12 +0,0 @@ -group: bbh_alt_pv_01_zeroshot -dataset_path: lukaemon/bbh -output_type: multiple_choice -test_split: test -doc_to_text: !function ../../styles.styles_01 -doc_to_target: !function ../../styles.doc_to_target -doc_to_choice: !function ../../styles.doc_to_choice -num_fewshot: 0 -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/boolean_expressions.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/boolean_expressions.yaml deleted file mode 100644 index aae4bfc1..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/boolean_expressions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "boolean_expressions" -"description": "Evaluate the result of a random Boolean expression.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_01_zeroshot_boolean_expressions" -"doc_to_target": target -"doc_to_choice": ["True", "False"] diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/causal_judgement.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/causal_judgement.yaml deleted file mode 100644 index 9dc5dfc3..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/causal_judgement.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "causal_judgement" -"description": "Answer questions about causal attribution.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_01_zeroshot_causal_judgement" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/date_understanding.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/date_understanding.yaml deleted file mode 100644 index 9cb5eef6..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/date_understanding.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "date_understanding" -"description": "Infer the date from context.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_01_zeroshot_date_understanding" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/disambiguation_qa.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/disambiguation_qa.yaml deleted file mode 100644 index df69dd71..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/disambiguation_qa.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "disambiguation_qa" -"description": "Clarify the meaning of sentences with ambiguous pronouns.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_01_zeroshot_disambiguation_qa" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/formal_fallacies.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/formal_fallacies.yaml deleted file mode 100644 index 34f384cd..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/formal_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_fallacies" -"description": "Distinguish deductively valid arguments from formal fallacies.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_01_zeroshot_formal_fallacies" -"doc_to_target": target -"doc_to_choice": ["valid", "invalid"] diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/geometric_shapes.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/geometric_shapes.yaml deleted file mode 100644 index 0a5d175e..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/geometric_shapes.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "geometric_shapes" -"description": "Name geometric shapes from their SVG paths.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_01_zeroshot_geometric_shapes" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/hyperbaton.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/hyperbaton.yaml deleted file mode 100644 index c51a7b93..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/hyperbaton.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "hyperbaton" -"description": "Order adjectives correctly in English sentences.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_01_zeroshot_hyperbaton" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/logical_deduction_five_objects.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/logical_deduction_five_objects.yaml deleted file mode 100644 index e9704906..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/logical_deduction_five_objects.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "logical_deduction_five_objects" -"description": "A logical deduction task which requires deducing the order of a sequence of objects.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_01_zeroshot_logical_deduction_five_objects" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/logical_deduction_seven_objects.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/logical_deduction_seven_objects.yaml deleted file mode 100644 index c33460c9..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/logical_deduction_seven_objects.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "logical_deduction_seven_objects" -"description": "A logical deduction task which requires deducing the order of a sequence of objects.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_01_zeroshot_logical_deduction_seven_objects" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/logical_deduction_three_objects.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/logical_deduction_three_objects.yaml deleted file mode 100644 index 2654f3a8..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/logical_deduction_three_objects.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "logical_deduction_three_objects" -"description": "A logical deduction task which requires deducing the order of a sequence of objects.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_01_zeroshot_logical_deduction_three_objects" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/movie_recommendation.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/movie_recommendation.yaml deleted file mode 100644 index f71fcc09..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/movie_recommendation.yaml +++ /dev/null @@ -1,5 +0,0 @@ -"dataset_name": "movie_recommendation" -"description": "Recommend movies similar to the given list of movies.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_01_zeroshot_movie_recommendation" -"process_docs": !function ../../utils.fix_movie_recommendation diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/navigate.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/navigate.yaml deleted file mode 100644 index ebd0a5b3..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/navigate.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "navigate" -"description": "Given a series of navigation instructions, determine whether one would end up back at the starting point.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_01_zeroshot_navigate" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/penguins_in_a_table.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/penguins_in_a_table.yaml deleted file mode 100644 index 490e45f6..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/penguins_in_a_table.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "penguins_in_a_table" -"description": "Answer questions about a table of penguins and their attributes.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_01_zeroshot_penguins_in_a_table" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/reasoning_about_colored_objects.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/reasoning_about_colored_objects.yaml deleted file mode 100644 index edbbb092..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/reasoning_about_colored_objects.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "reasoning_about_colored_objects" -"description": "Answer extremely simple questions about the colors of objects on a surface.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_01_zeroshot_reasoning_about_colored_objects" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/ruin_names.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/ruin_names.yaml deleted file mode 100644 index e391c296..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/ruin_names.yaml +++ /dev/null @@ -1,5 +0,0 @@ -"dataset_name": "ruin_names" -"description": "Select the humorous edit that 'ruins' the input movie or musical artist name.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_01_zeroshot_ruin_names" -"process_docs": !function ../../utils.fix_ruin_names diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/salient_translation_error_detection.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/salient_translation_error_detection.yaml deleted file mode 100644 index 40980d83..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/salient_translation_error_detection.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "salient_translation_error_detection" -"description": "Detect the type of error in an English translation of a German source sentence.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_01_zeroshot_salient_translation_error_detection" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/snarks.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/snarks.yaml deleted file mode 100644 index fe58da5d..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/snarks.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "snarks" -"description": "Determine which of two sentences is sarcastic.\n\nAccording to Cambridge University Dictionary, sarcasm is \"the use of remarks that clearly mean the opposite of what they say, made in order to hurt someone's feelings or to criticize something in a humorous way.\" Sarcastic sentences often contain satirical or ironic utterances, hyperboles, ambivalent or witty remarks.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_01_zeroshot_snarks" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/sports_understanding.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/sports_understanding.yaml deleted file mode 100644 index 8ed32a13..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/sports_understanding.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sports_understanding" -"description": "Determine whether an artificially constructed sentence relating to sports is plausible or not.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_01_zeroshot_sports_understanding" -"doc_to_target": target -"doc_to_choice": ["yes", "no"] diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/temporal_sequences.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/temporal_sequences.yaml deleted file mode 100644 index 60a5069d..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/temporal_sequences.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "temporal_sequences" -"description": "Task description: Answer questions about which times certain events could have occurred.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_01_zeroshot_temporal_sequences" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/tracking_shuffled_objects_five_objects.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/tracking_shuffled_objects_five_objects.yaml deleted file mode 100644 index 4df1d2fe..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/tracking_shuffled_objects_five_objects.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "tracking_shuffled_objects_five_objects" -"description": "A task requiring determining the final positions of a set of objects given their initial positions and a description of a sequence of swaps.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_01_zeroshot_tracking_shuffled_objects_five_objects" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/tracking_shuffled_objects_seven_objects.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/tracking_shuffled_objects_seven_objects.yaml deleted file mode 100644 index 7c710c67..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/tracking_shuffled_objects_seven_objects.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "tracking_shuffled_objects_seven_objects" -"description": "A task requiring determining the final positions of a set of objects given their initial positions and a description of a sequence of swaps.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_01_zeroshot_tracking_shuffled_objects_seven_objects" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/tracking_shuffled_objects_three_objects.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/tracking_shuffled_objects_three_objects.yaml deleted file mode 100644 index 7ca47942..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/tracking_shuffled_objects_three_objects.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "tracking_shuffled_objects_three_objects" -"description": "A task requiring determining the final positions of a set of objects given their initial positions and a description of a sequence of swaps.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_01_zeroshot_tracking_shuffled_objects_three_objects" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/web_of_lies.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/web_of_lies.yaml deleted file mode 100644 index b9dc118a..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/web_of_lies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "web_of_lies" -"description": "Evaluate a random boolean function expressed as a word problem.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_01_zeroshot_web_of_lies" -"doc_to_target": target -"doc_to_choice": ["Yes", "No"] diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/_zeroshot_template_yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/_zeroshot_template_yaml deleted file mode 100644 index 0ee7bf70..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/_zeroshot_template_yaml +++ /dev/null @@ -1,12 +0,0 @@ -group: bbh_alt_pv_02_zeroshot -dataset_path: lukaemon/bbh -output_type: multiple_choice -test_split: test -doc_to_text: !function ../../styles.styles_02 -doc_to_target: !function ../../styles.doc_to_target -doc_to_choice: !function ../../styles.doc_to_choice -num_fewshot: 0 -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/boolean_expressions.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/boolean_expressions.yaml deleted file mode 100644 index 383566b1..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/boolean_expressions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "boolean_expressions" -"description": "Evaluate the result of a random Boolean expression.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_02_zeroshot_boolean_expressions" -"doc_to_target": target -"doc_to_choice": ["True", "False"] diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/causal_judgement.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/causal_judgement.yaml deleted file mode 100644 index 5eba635d..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/causal_judgement.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "causal_judgement" -"description": "Answer questions about causal attribution.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_02_zeroshot_causal_judgement" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/date_understanding.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/date_understanding.yaml deleted file mode 100644 index 3e5a3e93..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/date_understanding.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "date_understanding" -"description": "Infer the date from context.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_02_zeroshot_date_understanding" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/disambiguation_qa.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/disambiguation_qa.yaml deleted file mode 100644 index a6d8c345..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/disambiguation_qa.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "disambiguation_qa" -"description": "Clarify the meaning of sentences with ambiguous pronouns.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_02_zeroshot_disambiguation_qa" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/formal_fallacies.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/formal_fallacies.yaml deleted file mode 100644 index 3f0b1975..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/formal_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_fallacies" -"description": "Distinguish deductively valid arguments from formal fallacies.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_02_zeroshot_formal_fallacies" -"doc_to_target": target -"doc_to_choice": ["valid", "invalid"] diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/geometric_shapes.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/geometric_shapes.yaml deleted file mode 100644 index 228567bb..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/geometric_shapes.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "geometric_shapes" -"description": "Name geometric shapes from their SVG paths.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_02_zeroshot_geometric_shapes" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/hyperbaton.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/hyperbaton.yaml deleted file mode 100644 index f79fa0ce..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/hyperbaton.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "hyperbaton" -"description": "Order adjectives correctly in English sentences.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_02_zeroshot_hyperbaton" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/logical_deduction_five_objects.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/logical_deduction_five_objects.yaml deleted file mode 100644 index bfdfdace..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/logical_deduction_five_objects.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "logical_deduction_five_objects" -"description": "A logical deduction task which requires deducing the order of a sequence of objects.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_02_zeroshot_logical_deduction_five_objects" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/logical_deduction_seven_objects.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/logical_deduction_seven_objects.yaml deleted file mode 100644 index bcca9b80..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/logical_deduction_seven_objects.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "logical_deduction_seven_objects" -"description": "A logical deduction task which requires deducing the order of a sequence of objects.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_02_zeroshot_logical_deduction_seven_objects" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/logical_deduction_three_objects.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/logical_deduction_three_objects.yaml deleted file mode 100644 index 327eea74..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/logical_deduction_three_objects.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "logical_deduction_three_objects" -"description": "A logical deduction task which requires deducing the order of a sequence of objects.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_02_zeroshot_logical_deduction_three_objects" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/movie_recommendation.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/movie_recommendation.yaml deleted file mode 100644 index 8ac66b5f..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/movie_recommendation.yaml +++ /dev/null @@ -1,5 +0,0 @@ -"dataset_name": "movie_recommendation" -"description": "Recommend movies similar to the given list of movies.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_02_zeroshot_movie_recommendation" -"process_docs": !function ../../utils.fix_movie_recommendation diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/navigate.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/navigate.yaml deleted file mode 100644 index d6442102..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/navigate.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "navigate" -"description": "Given a series of navigation instructions, determine whether one would end up back at the starting point.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_02_zeroshot_navigate" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/penguins_in_a_table.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/penguins_in_a_table.yaml deleted file mode 100644 index 186526da..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/penguins_in_a_table.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "penguins_in_a_table" -"description": "Answer questions about a table of penguins and their attributes.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_02_zeroshot_penguins_in_a_table" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/reasoning_about_colored_objects.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/reasoning_about_colored_objects.yaml deleted file mode 100644 index 25123984..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/reasoning_about_colored_objects.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "reasoning_about_colored_objects" -"description": "Answer extremely simple questions about the colors of objects on a surface.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_02_zeroshot_reasoning_about_colored_objects" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/ruin_names.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/ruin_names.yaml deleted file mode 100644 index bb5b7b52..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/ruin_names.yaml +++ /dev/null @@ -1,5 +0,0 @@ -"dataset_name": "ruin_names" -"description": "Select the humorous edit that 'ruins' the input movie or musical artist name.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_02_zeroshot_ruin_names" -"process_docs": !function ../../utils.fix_ruin_names diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/salient_translation_error_detection.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/salient_translation_error_detection.yaml deleted file mode 100644 index 85d8fb99..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/salient_translation_error_detection.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "salient_translation_error_detection" -"description": "Detect the type of error in an English translation of a German source sentence.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_02_zeroshot_salient_translation_error_detection" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/snarks.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/snarks.yaml deleted file mode 100644 index 9889128f..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/snarks.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "snarks" -"description": "Determine which of two sentences is sarcastic.\n\nAccording to Cambridge University Dictionary, sarcasm is \"the use of remarks that clearly mean the opposite of what they say, made in order to hurt someone's feelings or to criticize something in a humorous way.\" Sarcastic sentences often contain satirical or ironic utterances, hyperboles, ambivalent or witty remarks.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_02_zeroshot_snarks" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/sports_understanding.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/sports_understanding.yaml deleted file mode 100644 index a7c29ba4..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/sports_understanding.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sports_understanding" -"description": "Determine whether an artificially constructed sentence relating to sports is plausible or not.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_02_zeroshot_sports_understanding" -"doc_to_target": target -"doc_to_choice": ["yes", "no"] diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/temporal_sequences.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/temporal_sequences.yaml deleted file mode 100644 index 74b748f5..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/temporal_sequences.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "temporal_sequences" -"description": "Task description: Answer questions about which times certain events could have occurred.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_02_zeroshot_temporal_sequences" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/tracking_shuffled_objects_five_objects.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/tracking_shuffled_objects_five_objects.yaml deleted file mode 100644 index d9c00352..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/tracking_shuffled_objects_five_objects.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "tracking_shuffled_objects_five_objects" -"description": "A task requiring determining the final positions of a set of objects given their initial positions and a description of a sequence of swaps.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_02_zeroshot_tracking_shuffled_objects_five_objects" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/tracking_shuffled_objects_seven_objects.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/tracking_shuffled_objects_seven_objects.yaml deleted file mode 100644 index 89a5b785..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/tracking_shuffled_objects_seven_objects.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "tracking_shuffled_objects_seven_objects" -"description": "A task requiring determining the final positions of a set of objects given their initial positions and a description of a sequence of swaps.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_02_zeroshot_tracking_shuffled_objects_seven_objects" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/tracking_shuffled_objects_three_objects.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/tracking_shuffled_objects_three_objects.yaml deleted file mode 100644 index 6cb6b89a..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/tracking_shuffled_objects_three_objects.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "tracking_shuffled_objects_three_objects" -"description": "A task requiring determining the final positions of a set of objects given their initial positions and a description of a sequence of swaps.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_02_zeroshot_tracking_shuffled_objects_three_objects" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/web_of_lies.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/web_of_lies.yaml deleted file mode 100644 index 0be43b36..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/web_of_lies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "web_of_lies" -"description": "Evaluate a random boolean function expressed as a word problem.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_02_zeroshot_web_of_lies" -"doc_to_target": target -"doc_to_choice": ["Yes", "No"] diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/_zeroshot_template_yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/_zeroshot_template_yaml deleted file mode 100644 index 1e492baf..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/_zeroshot_template_yaml +++ /dev/null @@ -1,12 +0,0 @@ -group: bbh_alt_pv_03_zeroshot -dataset_path: lukaemon/bbh -output_type: multiple_choice -test_split: test -doc_to_text: !function ../../styles.styles_03 -doc_to_target: !function ../../styles.doc_to_target -doc_to_choice: !function ../../styles.doc_to_choice -num_fewshot: 0 -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/boolean_expressions.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/boolean_expressions.yaml deleted file mode 100644 index 27e9b1a4..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/boolean_expressions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "boolean_expressions" -"description": "Evaluate the result of a random Boolean expression.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_03_zeroshot_boolean_expressions" -"doc_to_target": target -"doc_to_choice": ["True", "False"] diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/causal_judgement.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/causal_judgement.yaml deleted file mode 100644 index 824ec01c..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/causal_judgement.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "causal_judgement" -"description": "Answer questions about causal attribution.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_03_zeroshot_causal_judgement" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/date_understanding.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/date_understanding.yaml deleted file mode 100644 index 8a93686c..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/date_understanding.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "date_understanding" -"description": "Infer the date from context.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_03_zeroshot_date_understanding" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/disambiguation_qa.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/disambiguation_qa.yaml deleted file mode 100644 index bc4324e4..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/disambiguation_qa.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "disambiguation_qa" -"description": "Clarify the meaning of sentences with ambiguous pronouns.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_03_zeroshot_disambiguation_qa" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/formal_fallacies.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/formal_fallacies.yaml deleted file mode 100644 index 42d8abcc..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/formal_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_fallacies" -"description": "Distinguish deductively valid arguments from formal fallacies.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_03_zeroshot_formal_fallacies" -"doc_to_target": target -"doc_to_choice": ["valid", "invalid"] diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/geometric_shapes.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/geometric_shapes.yaml deleted file mode 100644 index 719db41e..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/geometric_shapes.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "geometric_shapes" -"description": "Name geometric shapes from their SVG paths.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_03_zeroshot_geometric_shapes" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/hyperbaton.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/hyperbaton.yaml deleted file mode 100644 index 7905a343..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/hyperbaton.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "hyperbaton" -"description": "Order adjectives correctly in English sentences.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_03_zeroshot_hyperbaton" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/logical_deduction_five_objects.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/logical_deduction_five_objects.yaml deleted file mode 100644 index 14566a02..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/logical_deduction_five_objects.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "logical_deduction_five_objects" -"description": "A logical deduction task which requires deducing the order of a sequence of objects.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_03_zeroshot_logical_deduction_five_objects" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/logical_deduction_seven_objects.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/logical_deduction_seven_objects.yaml deleted file mode 100644 index 51ba44c8..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/logical_deduction_seven_objects.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "logical_deduction_seven_objects" -"description": "A logical deduction task which requires deducing the order of a sequence of objects.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_03_zeroshot_logical_deduction_seven_objects" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/logical_deduction_three_objects.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/logical_deduction_three_objects.yaml deleted file mode 100644 index bca7b49c..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/logical_deduction_three_objects.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "logical_deduction_three_objects" -"description": "A logical deduction task which requires deducing the order of a sequence of objects.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_03_zeroshot_logical_deduction_three_objects" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/movie_recommendation.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/movie_recommendation.yaml deleted file mode 100644 index 6e1f2ecb..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/movie_recommendation.yaml +++ /dev/null @@ -1,5 +0,0 @@ -"dataset_name": "movie_recommendation" -"description": "Recommend movies similar to the given list of movies.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_03_zeroshot_movie_recommendation" -"process_docs": !function ../../utils.fix_movie_recommendation diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/navigate.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/navigate.yaml deleted file mode 100644 index 393e364e..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/navigate.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "navigate" -"description": "Given a series of navigation instructions, determine whether one would end up back at the starting point.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_03_zeroshot_navigate" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/penguins_in_a_table.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/penguins_in_a_table.yaml deleted file mode 100644 index 71adb34b..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/penguins_in_a_table.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "penguins_in_a_table" -"description": "Answer questions about a table of penguins and their attributes.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_03_zeroshot_penguins_in_a_table" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/reasoning_about_colored_objects.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/reasoning_about_colored_objects.yaml deleted file mode 100644 index e1ddd9a4..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/reasoning_about_colored_objects.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "reasoning_about_colored_objects" -"description": "Answer extremely simple questions about the colors of objects on a surface.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_03_zeroshot_reasoning_about_colored_objects" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/ruin_names.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/ruin_names.yaml deleted file mode 100644 index e13bb3e6..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/ruin_names.yaml +++ /dev/null @@ -1,5 +0,0 @@ -"dataset_name": "ruin_names" -"description": "Select the humorous edit that 'ruins' the input movie or musical artist name.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_03_zeroshot_ruin_names" -"process_docs": !function ../../utils.fix_ruin_names diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/salient_translation_error_detection.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/salient_translation_error_detection.yaml deleted file mode 100644 index 86db045b..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/salient_translation_error_detection.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "salient_translation_error_detection" -"description": "Detect the type of error in an English translation of a German source sentence.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_03_zeroshot_salient_translation_error_detection" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/snarks.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/snarks.yaml deleted file mode 100644 index d8f2ca99..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/snarks.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "snarks" -"description": "Determine which of two sentences is sarcastic.\n\nAccording to Cambridge University Dictionary, sarcasm is \"the use of remarks that clearly mean the opposite of what they say, made in order to hurt someone's feelings or to criticize something in a humorous way.\" Sarcastic sentences often contain satirical or ironic utterances, hyperboles, ambivalent or witty remarks.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_03_zeroshot_snarks" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/sports_understanding.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/sports_understanding.yaml deleted file mode 100644 index cb3e70d6..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/sports_understanding.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sports_understanding" -"description": "Determine whether an artificially constructed sentence relating to sports is plausible or not.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_03_zeroshot_sports_understanding" -"doc_to_target": target -"doc_to_choice": ["yes", "no"] diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/temporal_sequences.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/temporal_sequences.yaml deleted file mode 100644 index bafde6b7..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/temporal_sequences.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "temporal_sequences" -"description": "Task description: Answer questions about which times certain events could have occurred.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_03_zeroshot_temporal_sequences" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/tracking_shuffled_objects_five_objects.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/tracking_shuffled_objects_five_objects.yaml deleted file mode 100644 index bec899a4..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/tracking_shuffled_objects_five_objects.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "tracking_shuffled_objects_five_objects" -"description": "A task requiring determining the final positions of a set of objects given their initial positions and a description of a sequence of swaps.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_03_zeroshot_tracking_shuffled_objects_five_objects" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/tracking_shuffled_objects_seven_objects.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/tracking_shuffled_objects_seven_objects.yaml deleted file mode 100644 index 0150b321..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/tracking_shuffled_objects_seven_objects.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "tracking_shuffled_objects_seven_objects" -"description": "A task requiring determining the final positions of a set of objects given their initial positions and a description of a sequence of swaps.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_03_zeroshot_tracking_shuffled_objects_seven_objects" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/tracking_shuffled_objects_three_objects.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/tracking_shuffled_objects_three_objects.yaml deleted file mode 100644 index 9e08ead9..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/tracking_shuffled_objects_three_objects.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"dataset_name": "tracking_shuffled_objects_three_objects" -"description": "A task requiring determining the final positions of a set of objects given their initial positions and a description of a sequence of swaps.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_03_zeroshot_tracking_shuffled_objects_three_objects" diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/web_of_lies.yaml b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/web_of_lies.yaml deleted file mode 100644 index 12ff9bc3..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/web_of_lies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "web_of_lies" -"description": "Evaluate a random boolean function expressed as a word problem.\n\n" -"include": "_zeroshot_template_yaml" -"task": "bbh_alt_pv_03_zeroshot_web_of_lies" -"doc_to_target": target -"doc_to_choice": ["Yes", "No"] diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/styles.py b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/styles.py deleted file mode 100644 index ae872179..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/styles.py +++ /dev/null @@ -1,72 +0,0 @@ -import re -import string - -yes_no = ["Yes", "No"] - - -def parse_choices(doc): - - input_text = doc["input"] - choice_string = input_text.split("Options:") - if len(choice_string) == 2: - choice_string = choice_string[-1] - if ("- Yes" in choice_string) and ("- No" in choice_string): - choices = yes_no - else: - choices = [ - c[4:].rstrip("\n") - for c in re.findall(r"\([A-Z]\) .*?\n|\([A-Z]\) .*?$", choice_string) - ] - return choices - else: - return [] - - -def styles_01(doc): - # Check for choices and remove them - choices = parse_choices(doc) - if choices != []: - doc_to_text = doc["input"].split("Options:")[0] - if doc_to_text[-1] in ["\n", " "]: - doc_to_text = doc_to_text[:-1] - else: - doc_to_text = doc["input"] - return doc_to_text - - -def styles_02(doc): - # Check for choices and remove them - doc_to_text = styles_01(doc) - return "Q: " + doc_to_text + "\nA:" - - -def styles_03(doc): - # Check for choices and remove them - doc_to_text = styles_01(doc) - return "Question: " + doc_to_text + "\nAnswer:" - - -def doc_to_choice(doc): - return parse_choices(doc) - - -def doc_to_target(doc): - target = doc["target"] - try: - if target in ["Yes", "No"]: - return yes_no.index(target) - else: - return string.ascii_uppercase.index(target[1:-1]) - # else: - # return parse_choices(doc).index(target) - - except Exception as err: - print("Full Doc") - print(doc) - print("Choices") - print(parse_choices(doc)) - print("Error") - print(err) - import sys - - sys.exit() diff --git a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/utils.py b/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/utils.py deleted file mode 100644 index 7cd31a48..00000000 --- a/lm_eval/tasks/bbh/alternative_worlds/prompt_variation/utils.py +++ /dev/null @@ -1,31 +0,0 @@ -# For fixing line 163 in `movie_recommendation` - - -def fix_movie_recommendation(data): - def _fix(doc): - if doc["target"] == "Monsters, Inc": - doc[ - "input" - ] = "Find a movie similar to Minority Report, Shrek, Catch Me If You Can, Aladdin:\nOptions:\n(A) Monsters, Inc\n(B) Children of the Night\n(C) The Incredible Shrinking Man\n(D) Town & Country" - doc["target"] = "(A)" - return doc - - return data.map(_fix) - - -def fix_ruin_names(data): - def _fix(doc): - if doc["target"] == "dearth, wind, & fire": - doc[ - "input" - ] = "Which of the following is a humorous edit of this artist or movie name: 'earth, wind, & fire'?\nOptions:\n(A) eareth, wind, & fire\n(B) earth, bind, & fire\n(C) earthm wind, & fire\n(D) dearth, wind, & fire" - doc["target"] = "(D)" - - elif doc["target"] == "rita, sue and bob poo": - doc[ - "input" - ] = "Which of the following is a humorous edit of this artist or movie name: 'rita, sue and bob too'?\nOptions:\n(A) rita, sue and bob too\n(B) rita, sue and bob poo\n(C) rita, sue and box too\n(D) rita,y sue and bob too" - doc["target"] = "(B)" - return doc - - return data.map(_fix) diff --git a/lm_eval/tasks/hellaswag/alternative_worlds/README.md b/lm_eval/tasks/hellaswag/alternative_worlds/README.md deleted file mode 100644 index 93600ae1..00000000 --- a/lm_eval/tasks/hellaswag/alternative_worlds/README.md +++ /dev/null @@ -1,20 +0,0 @@ - - -Investigate affect of letter options -- (A) -- A) -- A. -- A\t -- (a) -- a) -- a. -- a\t - -Answer types: -- letters only - - original option - - just letter -- letters + continuation - - original option - - just letter -- continuation diff --git a/lm_eval/tasks/hellaswag/alternative_worlds/_hellaswag_alt_yaml b/lm_eval/tasks/hellaswag/alternative_worlds/_hellaswag_alt_yaml deleted file mode 100644 index 8a88c7a1..00000000 --- a/lm_eval/tasks/hellaswag/alternative_worlds/_hellaswag_alt_yaml +++ /dev/null @@ -1,19 +0,0 @@ -dataset_path: Rowan/hellaswag -dataset_name: null -output_type: multiple_choice -training_split: train -validation_split: validation -test_split: null -process_docs: !function ../utils.process_docs -doc_to_text: "{{query}}" -doc_to_target: "{{label}}" -doc_to_choice: "{{choices}}" -metric_list: - - metric: acc - aggregation: mean - higher_is_better: true - - metric: acc_norm - aggregation: mean - higher_is_better: true - - metric: brier_score - higher_is_better: false diff --git a/lm_eval/tasks/hellaswag/alternative_worlds/hellaswag_alt.yaml b/lm_eval/tasks/hellaswag/alternative_worlds/hellaswag_alt.yaml deleted file mode 100644 index c60df999..00000000 --- a/lm_eval/tasks/hellaswag/alternative_worlds/hellaswag_alt.yaml +++ /dev/null @@ -1,10 +0,0 @@ -group: hellaswag_alt -task: - - hellaswag_01 - - hellaswag_02 - - hellaswag_03 - - hellaswag_04 - - hellaswag_05 - - hellaswag_06 - - hellaswag_07 - - hellaswag_08 diff --git a/lm_eval/tasks/hellaswag/alternative_worlds/style_01/a.yaml b/lm_eval/tasks/hellaswag/alternative_worlds/style_01/a.yaml deleted file mode 100644 index db54afc0..00000000 --- a/lm_eval/tasks/hellaswag/alternative_worlds/style_01/a.yaml +++ /dev/null @@ -1,7 +0,0 @@ -include: ../_hellaswag_alt_yaml -group: hellaswag_01 -group_alias: style_01 -task: hellaswag_01a -task_alias: a -doc_to_text: !function ../styles.template_01 -doc_to_choice: !function ../styles.choice_01a diff --git a/lm_eval/tasks/hellaswag/alternative_worlds/style_01/b.yaml b/lm_eval/tasks/hellaswag/alternative_worlds/style_01/b.yaml deleted file mode 100644 index 7c5890ca..00000000 --- a/lm_eval/tasks/hellaswag/alternative_worlds/style_01/b.yaml +++ /dev/null @@ -1,7 +0,0 @@ -include: ../_hellaswag_alt_yaml -group: hellaswag_01 -group_alias: style_01 -task: hellaswag_01b -task_alias: b -doc_to_text: !function ../styles.template_01 -doc_to_choice: !function ../styles.choice_01b diff --git a/lm_eval/tasks/hellaswag/alternative_worlds/style_01/c.yaml b/lm_eval/tasks/hellaswag/alternative_worlds/style_01/c.yaml deleted file mode 100644 index 2e7d61eb..00000000 --- a/lm_eval/tasks/hellaswag/alternative_worlds/style_01/c.yaml +++ /dev/null @@ -1,7 +0,0 @@ -include: ../_hellaswag_alt_yaml -group: hellaswag_01 -group_alias: style_01 -task: hellaswag_01c -task_alias: c -doc_to_text: !function ../styles.template_01 -doc_to_choice: !function ../styles.choice_01c diff --git a/lm_eval/tasks/hellaswag/alternative_worlds/style_02/a.yaml b/lm_eval/tasks/hellaswag/alternative_worlds/style_02/a.yaml deleted file mode 100644 index bae1a7c4..00000000 --- a/lm_eval/tasks/hellaswag/alternative_worlds/style_02/a.yaml +++ /dev/null @@ -1,7 +0,0 @@ -include: ../_hellaswag_alt_yaml -group: hellaswag_02 -group_alias: style_02 -task: hellaswag_02a -task_alias: a -doc_to_text: !function ../styles.template_02 -doc_to_choice: !function ../styles.choice_02a diff --git a/lm_eval/tasks/hellaswag/alternative_worlds/style_02/b.yaml b/lm_eval/tasks/hellaswag/alternative_worlds/style_02/b.yaml deleted file mode 100644 index b808d99b..00000000 --- a/lm_eval/tasks/hellaswag/alternative_worlds/style_02/b.yaml +++ /dev/null @@ -1,7 +0,0 @@ -include: ../_hellaswag_alt_yaml -group: hellaswag_02 -group_alias: style_02 -task: hellaswag_02b -task_alias: b -doc_to_text: !function ../styles.template_02 -doc_to_choice: !function ../styles.choice_02b diff --git a/lm_eval/tasks/hellaswag/alternative_worlds/style_02/c.yaml b/lm_eval/tasks/hellaswag/alternative_worlds/style_02/c.yaml deleted file mode 100644 index 3fff20f2..00000000 --- a/lm_eval/tasks/hellaswag/alternative_worlds/style_02/c.yaml +++ /dev/null @@ -1,7 +0,0 @@ -include: ../_hellaswag_alt_yaml -group: hellaswag_02 -group_alias: style_02 -task: hellaswag_02c -task_alias: c -doc_to_text: !function ../styles.template_02 -doc_to_choice: !function ../styles.choice_02c diff --git a/lm_eval/tasks/hellaswag/alternative_worlds/style_03/a.yaml b/lm_eval/tasks/hellaswag/alternative_worlds/style_03/a.yaml deleted file mode 100644 index 9b87ae71..00000000 --- a/lm_eval/tasks/hellaswag/alternative_worlds/style_03/a.yaml +++ /dev/null @@ -1,7 +0,0 @@ -include: ../_hellaswag_alt_yaml -group: hellaswag_03 -group_alias: style_03 -task: hellaswag_03a -task_alias: a -doc_to_text: !function ../styles.template_03 -doc_to_choice: !function ../styles.choice_03a diff --git a/lm_eval/tasks/hellaswag/alternative_worlds/style_03/b.yaml b/lm_eval/tasks/hellaswag/alternative_worlds/style_03/b.yaml deleted file mode 100644 index 1c93ffb3..00000000 --- a/lm_eval/tasks/hellaswag/alternative_worlds/style_03/b.yaml +++ /dev/null @@ -1,7 +0,0 @@ -include: ../_hellaswag_alt_yaml -group: hellaswag_03 -group_alias: style_03 -task: hellaswag_03b -task_alias: b -doc_to_text: !function ../styles.template_03 -doc_to_choice: !function ../styles.choice_03b diff --git a/lm_eval/tasks/hellaswag/alternative_worlds/style_03/c.yaml b/lm_eval/tasks/hellaswag/alternative_worlds/style_03/c.yaml deleted file mode 100644 index f33ccb84..00000000 --- a/lm_eval/tasks/hellaswag/alternative_worlds/style_03/c.yaml +++ /dev/null @@ -1,7 +0,0 @@ -include: ../_hellaswag_alt_yaml -group: hellaswag_03 -group_alias: style_03 -task: hellaswag_03c -task_alias: c -doc_to_text: !function ../styles.template_03 -doc_to_choice: !function ../styles.choice_03c diff --git a/lm_eval/tasks/hellaswag/alternative_worlds/style_04/a.yaml b/lm_eval/tasks/hellaswag/alternative_worlds/style_04/a.yaml deleted file mode 100644 index bb5bd68f..00000000 --- a/lm_eval/tasks/hellaswag/alternative_worlds/style_04/a.yaml +++ /dev/null @@ -1,7 +0,0 @@ -include: ../_hellaswag_alt_yaml -group: hellaswag_04 -group_alias: style_04 -task: hellaswag_04a -task_alias: a -doc_to_text: !function ../styles.template_04 -doc_to_choice: !function ../styles.choice_04a diff --git a/lm_eval/tasks/hellaswag/alternative_worlds/style_04/b.yaml b/lm_eval/tasks/hellaswag/alternative_worlds/style_04/b.yaml deleted file mode 100644 index 7a1e6156..00000000 --- a/lm_eval/tasks/hellaswag/alternative_worlds/style_04/b.yaml +++ /dev/null @@ -1,7 +0,0 @@ -include: ../_hellaswag_alt_yaml -group: hellaswag_04 -group_alias: style_04 -task: hellaswag_04b -task_alias: b -doc_to_text: !function ../styles.template_04 -doc_to_choice: !function ../styles.choice_04b diff --git a/lm_eval/tasks/hellaswag/alternative_worlds/style_04/c.yaml b/lm_eval/tasks/hellaswag/alternative_worlds/style_04/c.yaml deleted file mode 100644 index 0a30c7ab..00000000 --- a/lm_eval/tasks/hellaswag/alternative_worlds/style_04/c.yaml +++ /dev/null @@ -1,7 +0,0 @@ -include: ../_hellaswag_alt_yaml -group: hellaswag_04 -group_alias: style_04 -task: hellaswag_04c -task_alias: c -doc_to_text: !function ../styles.template_04 -doc_to_choice: !function ../styles.choice_04c diff --git a/lm_eval/tasks/hellaswag/alternative_worlds/style_05/a.yaml b/lm_eval/tasks/hellaswag/alternative_worlds/style_05/a.yaml deleted file mode 100644 index cd7b773b..00000000 --- a/lm_eval/tasks/hellaswag/alternative_worlds/style_05/a.yaml +++ /dev/null @@ -1,7 +0,0 @@ -include: ../_hellaswag_alt_yaml -group: hellaswag_05 -group_alias: style_05 -task: hellaswag_05a -task_alias: a -doc_to_text: !function ../styles.template_05 -doc_to_choice: !function ../styles.choice_05a diff --git a/lm_eval/tasks/hellaswag/alternative_worlds/style_05/b.yaml b/lm_eval/tasks/hellaswag/alternative_worlds/style_05/b.yaml deleted file mode 100644 index 4cbce6bd..00000000 --- a/lm_eval/tasks/hellaswag/alternative_worlds/style_05/b.yaml +++ /dev/null @@ -1,7 +0,0 @@ -include: ../_hellaswag_alt_yaml -group: hellaswag_05 -group_alias: style_05 -task: hellaswag_05b -task_alias: b -doc_to_text: !function ../styles.template_05 -doc_to_choice: !function ../styles.choice_05b diff --git a/lm_eval/tasks/hellaswag/alternative_worlds/style_05/c.yaml b/lm_eval/tasks/hellaswag/alternative_worlds/style_05/c.yaml deleted file mode 100644 index ca7e8a21..00000000 --- a/lm_eval/tasks/hellaswag/alternative_worlds/style_05/c.yaml +++ /dev/null @@ -1,7 +0,0 @@ -include: ../_hellaswag_alt_yaml -group: hellaswag_05 -group_alias: style_05 -task: hellaswag_05c -task_alias: c -doc_to_text: !function ../styles.template_05 -doc_to_choice: !function ../styles.choice_05c diff --git a/lm_eval/tasks/hellaswag/alternative_worlds/style_06/a.yaml b/lm_eval/tasks/hellaswag/alternative_worlds/style_06/a.yaml deleted file mode 100644 index 95a007b0..00000000 --- a/lm_eval/tasks/hellaswag/alternative_worlds/style_06/a.yaml +++ /dev/null @@ -1,7 +0,0 @@ -include: ../_hellaswag_alt_yaml -group: hellaswag_06 -group_alias: style_06 -task: hellaswag_06a -task_alias: a -doc_to_text: !function ../styles.template_06 -doc_to_choice: !function ../styles.choice_06a diff --git a/lm_eval/tasks/hellaswag/alternative_worlds/style_06/b.yaml b/lm_eval/tasks/hellaswag/alternative_worlds/style_06/b.yaml deleted file mode 100644 index 3df5d942..00000000 --- a/lm_eval/tasks/hellaswag/alternative_worlds/style_06/b.yaml +++ /dev/null @@ -1,7 +0,0 @@ -include: ../_hellaswag_alt_yaml -group: hellaswag_06 -group_alias: style_06 -task: hellaswag_06b -task_alias: b -doc_to_text: !function ../styles.template_06 -doc_to_choice: !function ../styles.choice_06b diff --git a/lm_eval/tasks/hellaswag/alternative_worlds/style_06/c.yaml b/lm_eval/tasks/hellaswag/alternative_worlds/style_06/c.yaml deleted file mode 100644 index a690f6e7..00000000 --- a/lm_eval/tasks/hellaswag/alternative_worlds/style_06/c.yaml +++ /dev/null @@ -1,7 +0,0 @@ -include: ../_hellaswag_alt_yaml -group: hellaswag_06 -group_alias: style_06 -task: hellaswag_06c -task_alias: c -doc_to_text: !function ../styles.template_06 -doc_to_choice: !function ../styles.choice_06c diff --git a/lm_eval/tasks/hellaswag/alternative_worlds/style_07/a.yaml b/lm_eval/tasks/hellaswag/alternative_worlds/style_07/a.yaml deleted file mode 100644 index 5d3d7329..00000000 --- a/lm_eval/tasks/hellaswag/alternative_worlds/style_07/a.yaml +++ /dev/null @@ -1,7 +0,0 @@ -include: ../_hellaswag_alt_yaml -group: hellaswag_07 -group_alias: style_07 -task: hellaswag_07a -task_alias: a -doc_to_text: !function ../styles.template_07 -doc_to_choice: !function ../styles.choice_07a diff --git a/lm_eval/tasks/hellaswag/alternative_worlds/style_07/b.yaml b/lm_eval/tasks/hellaswag/alternative_worlds/style_07/b.yaml deleted file mode 100644 index 3764e9e5..00000000 --- a/lm_eval/tasks/hellaswag/alternative_worlds/style_07/b.yaml +++ /dev/null @@ -1,7 +0,0 @@ -include: ../_hellaswag_alt_yaml -group: hellaswag_07 -group_alias: style_07 -task: hellaswag_07b -task_alias: b -doc_to_text: !function ../styles.template_07 -doc_to_choice: !function ../styles.choice_07b diff --git a/lm_eval/tasks/hellaswag/alternative_worlds/style_07/c.yaml b/lm_eval/tasks/hellaswag/alternative_worlds/style_07/c.yaml deleted file mode 100644 index 9be6f58c..00000000 --- a/lm_eval/tasks/hellaswag/alternative_worlds/style_07/c.yaml +++ /dev/null @@ -1,7 +0,0 @@ -include: ../_hellaswag_alt_yaml -group: hellaswag_07 -group_alias: style_07 -task: hellaswag_07c -task_alias: c -doc_to_text: !function ../styles.template_07 -doc_to_choice: !function ../styles.choice_07c diff --git a/lm_eval/tasks/hellaswag/alternative_worlds/style_08/a.yaml b/lm_eval/tasks/hellaswag/alternative_worlds/style_08/a.yaml deleted file mode 100644 index d7d5e465..00000000 --- a/lm_eval/tasks/hellaswag/alternative_worlds/style_08/a.yaml +++ /dev/null @@ -1,7 +0,0 @@ -include: ../_hellaswag_alt_yaml -group: hellaswag_08 -group_alias: style_08 -task: hellaswag_08a -task_alias: a -doc_to_text: !function ../styles.template_08 -doc_to_choice: !function ../styles.choice_08a diff --git a/lm_eval/tasks/hellaswag/alternative_worlds/style_08/b.yaml b/lm_eval/tasks/hellaswag/alternative_worlds/style_08/b.yaml deleted file mode 100644 index 7efccdd1..00000000 --- a/lm_eval/tasks/hellaswag/alternative_worlds/style_08/b.yaml +++ /dev/null @@ -1,7 +0,0 @@ -include: ../_hellaswag_alt_yaml -group: hellaswag_08 -group_alias: style_08 -task: hellaswag_08b -task_alias: b -doc_to_text: !function ../styles.template_08 -doc_to_choice: !function ../styles.choice_08b diff --git a/lm_eval/tasks/hellaswag/alternative_worlds/style_08/c.yaml b/lm_eval/tasks/hellaswag/alternative_worlds/style_08/c.yaml deleted file mode 100644 index 4642a3c4..00000000 --- a/lm_eval/tasks/hellaswag/alternative_worlds/style_08/c.yaml +++ /dev/null @@ -1,7 +0,0 @@ -include: ../_hellaswag_alt_yaml -group: hellaswag_08 -group_alias: style_08 -task: hellaswag_08c -task_alias: c -doc_to_text: !function ../styles.template_08 -doc_to_choice: !function ../styles.choice_08c diff --git a/lm_eval/tasks/hellaswag/alternative_worlds/styles.py b/lm_eval/tasks/hellaswag/alternative_worlds/styles.py deleted file mode 100644 index 4aa8c268..00000000 --- a/lm_eval/tasks/hellaswag/alternative_worlds/styles.py +++ /dev/null @@ -1,90 +0,0 @@ -import string -from functools import partial - - -def doc_to_text_base(alphabet, style, doc): - - choices = doc["choices"] - num = len(choices) - - letter_list = [style.format(letter) for letter in alphabet[0:num]] - - if "\t" in style: - choice_string = "{}{}" - else: - choice_string = "{} {}" - - doc_to_text = "\n\n".join( - [ - doc["query"] + "...", - " What is the most appropriate continuation?", - ] - + [choice_string.format(i, j) for i, j in zip(letter_list, choices)] - ) - - return doc_to_text - - -# Full continuation -def choice_A(doc): - return doc["choices"] - - -# Letters only -def choice_B(alphabet, style, doc): - - choices = doc["choices"] - num = len(choices) - - letter_list = [style.format(letter) for letter in alphabet[0:num]] - if "\t" in style: - letter_list = [letter.replace("\t", "") for letter in letter_list] - - return letter_list - - -# Letters + Full continuation -def choice_C(alphabet, style, doc): - - choices = doc["choices"] - num = len(choices) - - letter_list = [style.format(letter) for letter in alphabet[0:num]] - if "\t" not in style: - letter_list = [letter + " " for letter in letter_list] - - return [letter + choice for letter, choice in zip(letter_list, doc["choices"])] - - -template_01 = partial(doc_to_text_base, string.ascii_lowercase, "({})") -choice_01a = choice_A -choice_01b = partial(choice_B, string.ascii_lowercase, "({})") -choice_01c = partial(choice_C, string.ascii_lowercase, "({})") -template_02 = partial(doc_to_text_base, string.ascii_lowercase, "{})") -choice_02a = choice_A -choice_02b = partial(choice_B, string.ascii_lowercase, "{})") -choice_02c = partial(choice_C, string.ascii_lowercase, "{})") -template_03 = partial(doc_to_text_base, string.ascii_lowercase, "{}.") -choice_03a = choice_A -choice_03b = partial(choice_B, string.ascii_lowercase, "{}.") -choice_03c = partial(choice_C, string.ascii_lowercase, "{}.") -template_04 = partial(doc_to_text_base, string.ascii_lowercase, "{}\t") -choice_04a = choice_A -choice_04b = partial(choice_B, string.ascii_lowercase, "{}\t") -choice_04c = partial(choice_C, string.ascii_lowercase, "{}\t") -template_05 = partial(doc_to_text_base, string.ascii_uppercase, "({})") -choice_05a = choice_A -choice_05b = partial(choice_B, string.ascii_uppercase, "({})") -choice_05c = partial(choice_C, string.ascii_uppercase, "({})") -template_06 = partial(doc_to_text_base, string.ascii_uppercase, "{})") -choice_06a = choice_A -choice_06b = partial(choice_B, string.ascii_uppercase, "{})") -choice_06c = partial(choice_C, string.ascii_uppercase, "{})") -template_07 = partial(doc_to_text_base, string.ascii_uppercase, "{}.") -choice_07a = choice_A -choice_07b = partial(choice_B, string.ascii_uppercase, "{}.") -choice_07c = partial(choice_C, string.ascii_uppercase, "{}.") -template_08 = partial(doc_to_text_base, string.ascii_uppercase, "{}\t") -choice_08a = choice_A -choice_08b = partial(choice_B, string.ascii_uppercase, "{}\t") -choice_08c = partial(choice_C, string.ascii_uppercase, "{}\t") diff --git a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/README.md b/lm_eval/tasks/mathqa/alternative_worlds/output_variation/README.md deleted file mode 100644 index 6f89dda2..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/README.md +++ /dev/null @@ -1,18 +0,0 @@ - -Investigate affect of letter options -- (A) -- A) -- A. -- A\t -- (a) -- a) -- a. -- a\t -Answer types: -- letters only - - original option - - just letter -- letters + continuation - - original option - - just letter -- continuation diff --git a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/_mathqa_alt_yaml b/lm_eval/tasks/mathqa/alternative_worlds/output_variation/_mathqa_alt_yaml deleted file mode 100644 index cca78340..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/_mathqa_alt_yaml +++ /dev/null @@ -1,10 +0,0 @@ -dataset_path: math_qa -output_type: multiple_choice -training_split: train -validation_split: validation -test_split: test -doc_to_target: "{{['a', 'b', 'c', 'd', 'e'].index(correct)}}" -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/mathqa_alt.yaml b/lm_eval/tasks/mathqa/alternative_worlds/output_variation/mathqa_alt.yaml deleted file mode 100644 index d13679d1..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/mathqa_alt.yaml +++ /dev/null @@ -1,10 +0,0 @@ -group: mathqa_alt_ov -task: - - mathqa_alt_ov_01 - - mathqa_alt_ov_02 - - mathqa_alt_ov_03 - - mathqa_alt_ov_04 - - mathqa_alt_ov_05 - - mathqa_alt_ov_06 - - mathqa_alt_ov_07 - - mathqa_alt_ov_08 diff --git a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_01/a.yaml b/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_01/a.yaml deleted file mode 100644 index ca38bfd7..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_01/a.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_mathqa_alt_yaml -group: mathqa_alt_ov_01 -task: mathqa_alt_ov_01a -doc_to_text: !function ../styles.template_01 -doc_to_choice: !function ../styles.choice_01a diff --git a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_01/b.yaml b/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_01/b.yaml deleted file mode 100644 index ed2eae14..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_01/b.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_mathqa_alt_yaml -group: mathqa_alt_ov_01 -task: mathqa_alt_ov_01b -doc_to_text: !function ../styles.template_01 -doc_to_choice: !function ../styles.choice_01b diff --git a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_01/c.yaml b/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_01/c.yaml deleted file mode 100644 index 35a9d041..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_01/c.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_mathqa_alt_yaml -group: mathqa_alt_ov_01 -task: mathqa_alt_ov_01c -doc_to_text: !function ../styles.template_01 -doc_to_choice: !function ../styles.choice_01c diff --git a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_02/a.yaml b/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_02/a.yaml deleted file mode 100644 index d6a259f2..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_02/a.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_mathqa_alt_yaml -group: mathqa_alt_ov_02 -task: mathqa_alt_ov_02a -doc_to_text: !function ../styles.template_02 -doc_to_choice: !function ../styles.choice_02a diff --git a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_02/b.yaml b/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_02/b.yaml deleted file mode 100644 index 31ec5c1b..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_02/b.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_mathqa_alt_yaml -group: mathqa_alt_ov_02 -task: mathqa_alt_ov_02b -doc_to_text: !function ../styles.template_02 -doc_to_choice: !function ../styles.choice_02b diff --git a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_02/c.yaml b/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_02/c.yaml deleted file mode 100644 index 4a5e1dfa..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_02/c.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_mathqa_alt_yaml -group: mathqa_alt_ov_02 -task: mathqa_alt_ov_02c -doc_to_text: !function ../styles.template_02 -doc_to_choice: !function ../styles.choice_02c diff --git a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_03/a.yaml b/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_03/a.yaml deleted file mode 100644 index 22c1466c..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_03/a.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_mathqa_alt_yaml -group: mathqa_alt_ov_03 -task: mathqa_alt_ov_03a -doc_to_text: !function ../styles.template_03 -doc_to_choice: !function ../styles.choice_03a diff --git a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_03/b.yaml b/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_03/b.yaml deleted file mode 100644 index 83ae0a0e..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_03/b.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_mathqa_alt_yaml -group: mathqa_alt_ov_03 -task: mathqa_alt_ov_03b -doc_to_text: !function ../styles.template_03 -doc_to_choice: !function ../styles.choice_03b diff --git a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_03/c.yaml b/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_03/c.yaml deleted file mode 100644 index f9bb436f..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_03/c.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_mathqa_alt_yaml -group: mathqa_alt_ov_03 -task: mathqa_alt_ov_03c -doc_to_text: !function ../styles.template_03 -doc_to_choice: !function ../styles.choice_03c diff --git a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_04/a.yaml b/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_04/a.yaml deleted file mode 100644 index fa9f1ee7..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_04/a.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_mathqa_alt_yaml -group: mathqa_alt_ov_04 -task: mathqa_alt_ov_04a -doc_to_text: !function ../styles.template_04 -doc_to_choice: !function ../styles.choice_04a diff --git a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_04/b.yaml b/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_04/b.yaml deleted file mode 100644 index b68fa2b4..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_04/b.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_mathqa_alt_yaml -group: mathqa_alt_ov_04 -task: mathqa_alt_ov_04b -doc_to_text: !function ../styles.template_04 -doc_to_choice: !function ../styles.choice_04b diff --git a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_04/c.yaml b/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_04/c.yaml deleted file mode 100644 index d7f8eb20..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_04/c.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_mathqa_alt_yaml -group: mathqa_alt_ov_04 -task: mathqa_alt_ov_04c -doc_to_text: !function ../styles.template_04 -doc_to_choice: !function ../styles.choice_04c diff --git a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_05/a.yaml b/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_05/a.yaml deleted file mode 100644 index 8431baa5..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_05/a.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_mathqa_alt_yaml -group: mathqa_alt_ov_05 -task: mathqa_alt_ov_05a -doc_to_text: !function ../styles.template_05 -doc_to_choice: !function ../styles.choice_05a diff --git a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_05/b.yaml b/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_05/b.yaml deleted file mode 100644 index b13d6479..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_05/b.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_mathqa_alt_yaml -group: mathqa_alt_ov_05 -task: mathqa_alt_ov_05b -doc_to_text: !function ../styles.template_05 -doc_to_choice: !function ../styles.choice_05b diff --git a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_05/c.yaml b/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_05/c.yaml deleted file mode 100644 index fb23a69d..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_05/c.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_mathqa_alt_yaml -group: mathqa_alt_ov_05 -task: mathqa_alt_ov_05c -doc_to_text: !function ../styles.template_05 -doc_to_choice: !function ../styles.choice_05c diff --git a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_06/a.yaml b/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_06/a.yaml deleted file mode 100644 index 99e8c27a..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_06/a.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_mathqa_alt_yaml -group: mathqa_alt_ov_06 -task: mathqa_alt_ov_06a -doc_to_text: !function ../styles.template_06 -doc_to_choice: !function ../styles.choice_06a diff --git a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_06/b.yaml b/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_06/b.yaml deleted file mode 100644 index 77fd1d62..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_06/b.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_mathqa_alt_yaml -group: mathqa_alt_ov_06 -task: mathqa_alt_ov_06b -doc_to_text: !function ../styles.template_06 -doc_to_choice: !function ../styles.choice_06b diff --git a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_06/c.yaml b/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_06/c.yaml deleted file mode 100644 index af60a24e..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_06/c.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_mathqa_alt_yaml -group: mathqa_alt_ov_06 -task: mathqa_alt_ov_06c -doc_to_text: !function ../styles.template_06 -doc_to_choice: !function ../styles.choice_06c diff --git a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_07/a.yaml b/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_07/a.yaml deleted file mode 100644 index 00a81155..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_07/a.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_mathqa_alt_yaml -group: mathqa_alt_ov_07 -task: mathqa_alt_ov_07a -doc_to_text: !function ../styles.template_07 -doc_to_choice: !function ../styles.choice_07a diff --git a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_07/b.yaml b/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_07/b.yaml deleted file mode 100644 index 6f9e1b2a..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_07/b.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_mathqa_alt_yaml -group: mathqa_alt_ov_07 -task: mathqa_alt_ov_07b -doc_to_text: !function ../styles.template_07 -doc_to_choice: !function ../styles.choice_07b diff --git a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_07/c.yaml b/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_07/c.yaml deleted file mode 100644 index 0b40d282..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_07/c.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_mathqa_alt_yaml -group: mathqa_alt_ov_07 -task: mathqa_alt_ov_07c -doc_to_text: !function ../styles.template_07 -doc_to_choice: !function ../styles.choice_07c diff --git a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_08/a.yaml b/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_08/a.yaml deleted file mode 100644 index f2bb3788..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_08/a.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_mathqa_alt_yaml -group: mathqa_alt_ov_08 -task: mathqa_alt_ov_08a -doc_to_text: !function ../styles.template_08 -doc_to_choice: !function ../styles.choice_08a diff --git a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_08/b.yaml b/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_08/b.yaml deleted file mode 100644 index 4b3b2a67..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_08/b.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_mathqa_alt_yaml -group: mathqa_alt_ov_08 -task: mathqa_alt_ov_08b -doc_to_text: !function ../styles.template_08 -doc_to_choice: !function ../styles.choice_08b diff --git a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_08/c.yaml b/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_08/c.yaml deleted file mode 100644 index 2242027d..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_08/c.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_mathqa_alt_yaml -group: mathqa_alt_ov_08 -task: mathqa_alt_ov_08c -doc_to_text: !function ../styles.template_08 -doc_to_choice: !function ../styles.choice_08c diff --git a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/styles.py b/lm_eval/tasks/mathqa/alternative_worlds/output_variation/styles.py deleted file mode 100644 index 09bc91f3..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/output_variation/styles.py +++ /dev/null @@ -1,86 +0,0 @@ -import re -import string -from functools import partial - - -def parse_choices(doc): - choices = [ - c[4:].rstrip(" ,") - for c in re.findall(r"[abcd] \) .*?, |e \) .*?$", doc["options"]) - ] - return choices - - -def doc_to_text_base(alphabet, style, doc): - choices = parse_choices(doc) - num = len(choices) - letter_list = [style.format(letter) for letter in alphabet[0:num]] - if "\t" in style: - choice_string = "{}{}" - else: - choice_string = "{} {}" - doc_to_text = "\n".join( - ["Question: " + doc["Problem"]] - + [choice_string.format(i, j) for i, j in zip(letter_list, choices)] - + ["Answer:"] - ) - return doc_to_text - - -# Full continuation -def choice_A(doc): - return parse_choices(doc) - - -# Letters only -def choice_B(alphabet, style, doc): - choices = parse_choices(doc) - num = len(choices) - letter_list = [style.format(letter) for letter in alphabet[0:num]] - if "\t" in style: - letter_list = [letter.replace("\t", "") for letter in letter_list] - return letter_list - - -# Letters + Full continuation -def choice_C(alphabet, style, doc): - choices = parse_choices(doc) - num = len(choices) - letter_list = [style.format(letter) for letter in alphabet[0:num]] - if "\t" not in style: - letter_list = [letter + " " for letter in letter_list] - return [letter + choice for letter, choice in zip(letter_list, choices)] - - -template_01 = partial(doc_to_text_base, string.ascii_lowercase, "({})") -choice_01a = choice_A -choice_01b = partial(choice_B, string.ascii_lowercase, "({})") -choice_01c = partial(choice_C, string.ascii_lowercase, "({})") -template_02 = partial(doc_to_text_base, string.ascii_lowercase, "{})") -choice_02a = choice_A -choice_02b = partial(choice_B, string.ascii_lowercase, "{})") -choice_02c = partial(choice_C, string.ascii_lowercase, "{})") -template_03 = partial(doc_to_text_base, string.ascii_lowercase, "{}.") -choice_03a = choice_A -choice_03b = partial(choice_B, string.ascii_lowercase, "{}.") -choice_03c = partial(choice_C, string.ascii_lowercase, "{}.") -template_04 = partial(doc_to_text_base, string.ascii_lowercase, "{}\t") -choice_04a = choice_A -choice_04b = partial(choice_B, string.ascii_lowercase, "{}\t") -choice_04c = partial(choice_C, string.ascii_lowercase, "{}\t") -template_05 = partial(doc_to_text_base, string.ascii_uppercase, "({})") -choice_05a = choice_A -choice_05b = partial(choice_B, string.ascii_uppercase, "({})") -choice_05c = partial(choice_C, string.ascii_uppercase, "({})") -template_06 = partial(doc_to_text_base, string.ascii_uppercase, "{})") -choice_06a = choice_A -choice_06b = partial(choice_B, string.ascii_uppercase, "{})") -choice_06c = partial(choice_C, string.ascii_uppercase, "{})") -template_07 = partial(doc_to_text_base, string.ascii_uppercase, "{}.") -choice_07a = choice_A -choice_07b = partial(choice_B, string.ascii_uppercase, "{}.") -choice_07c = partial(choice_C, string.ascii_uppercase, "{}.") -template_08 = partial(doc_to_text_base, string.ascii_uppercase, "{}\t") -choice_08a = choice_A -choice_08b = partial(choice_B, string.ascii_uppercase, "{}\t") -choice_08c = partial(choice_C, string.ascii_uppercase, "{}\t") diff --git a/lm_eval/tasks/mathqa/alternative_worlds/prompt_variation/style_01.yaml b/lm_eval/tasks/mathqa/alternative_worlds/prompt_variation/style_01.yaml deleted file mode 100644 index 0c91f8e3..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/prompt_variation/style_01.yaml +++ /dev/null @@ -1,16 +0,0 @@ -group: mathqa_alt_pv -task: mathqa_alt_pv_01 -dataset_path: math_qa -output_type: multiple_choice -training_split: train -validation_split: validation -test_split: test -doc_to_text: "{{Problem}}" -doc_to_target: "{{['a', 'b', 'c', 'd', 'e'].index(correct)}}" -doc_to_choice: !function ../../utils.doc_to_choice -should_decontaminate: true -doc_to_decontamination_query: "{{Problem}}" -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mathqa/alternative_worlds/prompt_variation/style_02.yaml b/lm_eval/tasks/mathqa/alternative_worlds/prompt_variation/style_02.yaml deleted file mode 100644 index 16183cc3..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/prompt_variation/style_02.yaml +++ /dev/null @@ -1,16 +0,0 @@ -group: mathqa_alt_pv -task: mathqa_alt_pv_02 -dataset_path: math_qa -output_type: multiple_choice -training_split: train -validation_split: validation -test_split: test -doc_to_text: "Q: {{Problem}}\nA:" -doc_to_target: "{{['a', 'b', 'c', 'd', 'e'].index(correct)}}" -doc_to_choice: !function ../../utils.doc_to_choice -should_decontaminate: true -doc_to_decontamination_query: "Q: {{Problem}}\nA:" -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mathqa/alternative_worlds/prompt_variation/style_03.yaml b/lm_eval/tasks/mathqa/alternative_worlds/prompt_variation/style_03.yaml deleted file mode 100644 index c757e6d2..00000000 --- a/lm_eval/tasks/mathqa/alternative_worlds/prompt_variation/style_03.yaml +++ /dev/null @@ -1,16 +0,0 @@ -group: mathqa_alt_pv -task: mathqa_alt_pv_03 -dataset_path: math_qa -output_type: multiple_choice -training_split: train -validation_split: validation -test_split: test -doc_to_text: "Question: {{Problem}}\nAnswer:" -doc_to_target: "{{['a', 'b', 'c', 'd', 'e'].index(correct)}}" -doc_to_choice: !function ../../utils.doc_to_choice -should_decontaminate: true -doc_to_decontamination_query: "Question: {{Problem}}\nAnswer:" -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mmlu/alternative_worlds/mmlu_output_variation.yaml b/lm_eval/tasks/mmlu/alternative_worlds/mmlu_output_variation.yaml deleted file mode 100644 index 28c5caa9..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/mmlu_output_variation.yaml +++ /dev/null @@ -1,10 +0,0 @@ -group: mmlu_alt_ov -task: - - mmlu_alt_ov_01 - - mmlu_alt_ov_02 - - mmlu_alt_ov_03 - - mmlu_alt_ov_04 - - mmlu_alt_ov_05 - - mmlu_alt_ov_06 - - mmlu_alt_ov_07 - - mmlu_alt_ov_08 diff --git a/lm_eval/tasks/mmlu/alternative_worlds/mmlu_prompt_variation.yaml b/lm_eval/tasks/mmlu/alternative_worlds/mmlu_prompt_variation.yaml deleted file mode 100644 index d210cd14..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/mmlu_prompt_variation.yaml +++ /dev/null @@ -1,5 +0,0 @@ -group: mmlu_alt_pv -task: - - mmlu_alt_pv_01 - - mmlu_alt_pv_02 - - mmlu_alt_pv_03 diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/_mmlu_ov_01.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/_mmlu_ov_01.yaml deleted file mode 100644 index 80dc2866..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/_mmlu_ov_01.yaml +++ /dev/null @@ -1,5 +0,0 @@ -group: mmlu_alt_ov_01 -task: - - mmlu_alt_ov_01a - - mmlu_alt_ov_01b - - mmlu_alt_ov_01c diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/_mmlu.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/_mmlu.yaml deleted file mode 100644 index 2839307e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/_mmlu.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: mmlu_alt_ov_01a -task: - - mmlu_alt_ov_01a_stem - - mmlu_alt_ov_01a_other - - mmlu_alt_ov_01a_social_sciences - - mmlu_alt_ov_01a_humanities diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/_template_yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/_template_yaml deleted file mode 100644 index ffd6d617..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/_template_yaml +++ /dev/null @@ -1,11 +0,0 @@ -dataset_path: hails/mmlu_no_train -test_split: test -fewshot_split: dev -output_type: multiple_choice -doc_to_text: !function ../../../styles.template_01 -doc_to_choice: !function ../../../styles.choice_01a -doc_to_target: answer -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_abstract_algebra.yaml deleted file mode 100644 index 8b6d983d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_abstract_algebra.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "abstract_algebra" -"description": "The following are multiple choice questions (with answers) about abstract\ - \ algebra.\n\n" -"group": "mmlu_alt_ov_01a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_anatomy.yaml deleted file mode 100644 index 8179c42e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_anatomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "anatomy" -"description": "The following are multiple choice questions (with answers) about anatomy.\n\ - \n" -"group": "mmlu_alt_ov_01a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_anatomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_astronomy.yaml deleted file mode 100644 index d81a6206..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_astronomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "astronomy" -"description": "The following are multiple choice questions (with answers) about astronomy.\n\ - \n" -"group": "mmlu_alt_ov_01a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_astronomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_business_ethics.yaml deleted file mode 100644 index b435f2d1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_business_ethics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "business_ethics" -"description": "The following are multiple choice questions (with answers) about business\ - \ ethics.\n\n" -"group": "mmlu_alt_ov_01a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_business_ethics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_clinical_knowledge.yaml deleted file mode 100644 index 397f1a22..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_clinical_knowledge.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "clinical_knowledge" -"description": "The following are multiple choice questions (with answers) about clinical\ - \ knowledge.\n\n" -"group": "mmlu_alt_ov_01a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_college_biology.yaml deleted file mode 100644 index b56ece83..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_college_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_biology" -"description": "The following are multiple choice questions (with answers) about college\ - \ biology.\n\n" -"group": "mmlu_alt_ov_01a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_college_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_college_chemistry.yaml deleted file mode 100644 index 200619e1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_college_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_chemistry" -"description": "The following are multiple choice questions (with answers) about college\ - \ chemistry.\n\n" -"group": "mmlu_alt_ov_01a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_college_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_college_computer_science.yaml deleted file mode 100644 index b3c603c5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_college_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_computer_science" -"description": "The following are multiple choice questions (with answers) about college\ - \ computer science.\n\n" -"group": "mmlu_alt_ov_01a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_college_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_college_mathematics.yaml deleted file mode 100644 index 0ee8bff0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_college_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_mathematics" -"description": "The following are multiple choice questions (with answers) about college\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_01a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_college_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_college_medicine.yaml deleted file mode 100644 index 4edd910e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_college_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_medicine" -"description": "The following are multiple choice questions (with answers) about college\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_01a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_college_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_college_physics.yaml deleted file mode 100644 index 6f57ec94..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_college_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_physics" -"description": "The following are multiple choice questions (with answers) about college\ - \ physics.\n\n" -"group": "mmlu_alt_ov_01a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_college_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_computer_security.yaml deleted file mode 100644 index 1af0a156..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_computer_security.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "computer_security" -"description": "The following are multiple choice questions (with answers) about computer\ - \ security.\n\n" -"group": "mmlu_alt_ov_01a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_computer_security" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_conceptual_physics.yaml deleted file mode 100644 index 6b135195..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_conceptual_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "conceptual_physics" -"description": "The following are multiple choice questions (with answers) about conceptual\ - \ physics.\n\n" -"group": "mmlu_alt_ov_01a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_econometrics.yaml deleted file mode 100644 index 9ebd1493..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_econometrics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "econometrics" -"description": "The following are multiple choice questions (with answers) about econometrics.\n\ - \n" -"group": "mmlu_alt_ov_01a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_econometrics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_electrical_engineering.yaml deleted file mode 100644 index 1aca0f3e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_electrical_engineering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "electrical_engineering" -"description": "The following are multiple choice questions (with answers) about electrical\ - \ engineering.\n\n" -"group": "mmlu_alt_ov_01a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_elementary_mathematics.yaml deleted file mode 100644 index 027d0eb4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_elementary_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "elementary_mathematics" -"description": "The following are multiple choice questions (with answers) about elementary\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_01a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_formal_logic.yaml deleted file mode 100644 index ec8595a6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_formal_logic.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_logic" -"description": "The following are multiple choice questions (with answers) about formal\ - \ logic.\n\n" -"group": "mmlu_alt_ov_01a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_formal_logic" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_global_facts.yaml deleted file mode 100644 index dd4cbb42..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_global_facts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "global_facts" -"description": "The following are multiple choice questions (with answers) about global\ - \ facts.\n\n" -"group": "mmlu_alt_ov_01a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_global_facts" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_biology.yaml deleted file mode 100644 index 23a6c360..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_biology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school biology.\n\n" -"group": "mmlu_alt_ov_01a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_high_school_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_chemistry.yaml deleted file mode 100644 index 64224c37..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_chemistry" -"description": "The following are multiple choice questions (with answers) about high\ - \ school chemistry.\n\n" -"group": "mmlu_alt_ov_01a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_computer_science.yaml deleted file mode 100644 index 5a5766ea..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_computer_science" -"description": "The following are multiple choice questions (with answers) about high\ - \ school computer science.\n\n" -"group": "mmlu_alt_ov_01a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_european_history.yaml deleted file mode 100644 index 48d73dbd..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_european_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_european_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school european history.\n\n" -"group": "mmlu_alt_ov_01a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_geography.yaml deleted file mode 100644 index bcce14e7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_geography.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_geography" -"description": "The following are multiple choice questions (with answers) about high\ - \ school geography.\n\n" -"group": "mmlu_alt_ov_01a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_high_school_geography" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_government_and_politics.yaml deleted file mode 100644 index be05b188..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_government_and_politics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_government_and_politics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school government and politics.\n\n" -"group": "mmlu_alt_ov_01a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_macroeconomics.yaml deleted file mode 100644 index c1e46832..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_macroeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_macroeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school macroeconomics.\n\n" -"group": "mmlu_alt_ov_01a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_mathematics.yaml deleted file mode 100644 index 19526617..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_mathematics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school mathematics.\n\n" -"group": "mmlu_alt_ov_01a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_microeconomics.yaml deleted file mode 100644 index 6d6e1236..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_microeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_microeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school microeconomics.\n\n" -"group": "mmlu_alt_ov_01a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_physics.yaml deleted file mode 100644 index 258b3a99..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_physics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school physics.\n\n" -"group": "mmlu_alt_ov_01a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_high_school_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_psychology.yaml deleted file mode 100644 index 975a9128..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_psychology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school psychology.\n\n" -"group": "mmlu_alt_ov_01a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_statistics.yaml deleted file mode 100644 index 47e394d0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_statistics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_statistics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school statistics.\n\n" -"group": "mmlu_alt_ov_01a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_us_history.yaml deleted file mode 100644 index 034e88e4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_us_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_us_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school us history.\n\n" -"group": "mmlu_alt_ov_01a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_world_history.yaml deleted file mode 100644 index 67aa7e65..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_high_school_world_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_world_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school world history.\n\n" -"group": "mmlu_alt_ov_01a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_human_aging.yaml deleted file mode 100644 index d165939c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_human_aging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_aging" -"description": "The following are multiple choice questions (with answers) about human\ - \ aging.\n\n" -"group": "mmlu_alt_ov_01a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_human_aging" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_human_sexuality.yaml deleted file mode 100644 index 7f6db037..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_human_sexuality.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_sexuality" -"description": "The following are multiple choice questions (with answers) about human\ - \ sexuality.\n\n" -"group": "mmlu_alt_ov_01a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_human_sexuality" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_international_law.yaml deleted file mode 100644 index 96c4c5bb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_international_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "international_law" -"description": "The following are multiple choice questions (with answers) about international\ - \ law.\n\n" -"group": "mmlu_alt_ov_01a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_international_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_jurisprudence.yaml deleted file mode 100644 index 6a64fd5f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_jurisprudence.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "jurisprudence" -"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ - \n" -"group": "mmlu_alt_ov_01a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_jurisprudence" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_logical_fallacies.yaml deleted file mode 100644 index c56f545c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_logical_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "logical_fallacies" -"description": "The following are multiple choice questions (with answers) about logical\ - \ fallacies.\n\n" -"group": "mmlu_alt_ov_01a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_machine_learning.yaml deleted file mode 100644 index 5624ee9c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_machine_learning.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "machine_learning" -"description": "The following are multiple choice questions (with answers) about machine\ - \ learning.\n\n" -"group": "mmlu_alt_ov_01a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_machine_learning" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_management.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_management.yaml deleted file mode 100644 index 23cd77ca..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_management.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "management" -"description": "The following are multiple choice questions (with answers) about management.\n\ - \n" -"group": "mmlu_alt_ov_01a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_management" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_marketing.yaml deleted file mode 100644 index 8f16d837..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_marketing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "marketing" -"description": "The following are multiple choice questions (with answers) about marketing.\n\ - \n" -"group": "mmlu_alt_ov_01a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_marketing" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_medical_genetics.yaml deleted file mode 100644 index 36b8beeb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_medical_genetics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "medical_genetics" -"description": "The following are multiple choice questions (with answers) about medical\ - \ genetics.\n\n" -"group": "mmlu_alt_ov_01a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_medical_genetics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_miscellaneous.yaml deleted file mode 100644 index 14d35704..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_miscellaneous.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "miscellaneous" -"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ - \n" -"group": "mmlu_alt_ov_01a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_miscellaneous" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_moral_disputes.yaml deleted file mode 100644 index 2ffc9177..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_moral_disputes.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_disputes" -"description": "The following are multiple choice questions (with answers) about moral\ - \ disputes.\n\n" -"group": "mmlu_alt_ov_01a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_moral_disputes" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_moral_scenarios.yaml deleted file mode 100644 index 08de040c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_moral_scenarios.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_scenarios" -"description": "The following are multiple choice questions (with answers) about moral\ - \ scenarios.\n\n" -"group": "mmlu_alt_ov_01a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_nutrition.yaml deleted file mode 100644 index e3729066..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_nutrition.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "nutrition" -"description": "The following are multiple choice questions (with answers) about nutrition.\n\ - \n" -"group": "mmlu_alt_ov_01a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_nutrition" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_philosophy.yaml deleted file mode 100644 index 446ae69e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_philosophy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "philosophy" -"description": "The following are multiple choice questions (with answers) about philosophy.\n\ - \n" -"group": "mmlu_alt_ov_01a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_philosophy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_prehistory.yaml deleted file mode 100644 index e918b8bc..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_prehistory.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "prehistory" -"description": "The following are multiple choice questions (with answers) about prehistory.\n\ - \n" -"group": "mmlu_alt_ov_01a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_prehistory" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_professional_accounting.yaml deleted file mode 100644 index d88699a4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_professional_accounting.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_accounting" -"description": "The following are multiple choice questions (with answers) about professional\ - \ accounting.\n\n" -"group": "mmlu_alt_ov_01a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_professional_accounting" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_professional_law.yaml deleted file mode 100644 index 4720ea2d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_professional_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_law" -"description": "The following are multiple choice questions (with answers) about professional\ - \ law.\n\n" -"group": "mmlu_alt_ov_01a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_professional_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_professional_medicine.yaml deleted file mode 100644 index 94c23c29..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_professional_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_medicine" -"description": "The following are multiple choice questions (with answers) about professional\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_01a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_professional_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_professional_psychology.yaml deleted file mode 100644 index 0eba1445..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_professional_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_psychology" -"description": "The following are multiple choice questions (with answers) about professional\ - \ psychology.\n\n" -"group": "mmlu_alt_ov_01a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_professional_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_public_relations.yaml deleted file mode 100644 index 638362e4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_public_relations.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "public_relations" -"description": "The following are multiple choice questions (with answers) about public\ - \ relations.\n\n" -"group": "mmlu_alt_ov_01a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_public_relations" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_security_studies.yaml deleted file mode 100644 index d14b4ef4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_security_studies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "security_studies" -"description": "The following are multiple choice questions (with answers) about security\ - \ studies.\n\n" -"group": "mmlu_alt_ov_01a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_security_studies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_sociology.yaml deleted file mode 100644 index 41969f6d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_sociology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sociology" -"description": "The following are multiple choice questions (with answers) about sociology.\n\ - \n" -"group": "mmlu_alt_ov_01a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_sociology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_us_foreign_policy.yaml deleted file mode 100644 index 02380c19..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_us_foreign_policy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "us_foreign_policy" -"description": "The following are multiple choice questions (with answers) about us\ - \ foreign policy.\n\n" -"group": "mmlu_alt_ov_01a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_virology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_virology.yaml deleted file mode 100644 index f077a31a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_virology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "virology" -"description": "The following are multiple choice questions (with answers) about virology.\n\ - \n" -"group": "mmlu_alt_ov_01a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_virology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_world_religions.yaml deleted file mode 100644 index c043a278..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_world_religions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "world_religions" -"description": "The following are multiple choice questions (with answers) about world\ - \ religions.\n\n" -"group": "mmlu_alt_ov_01a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01a_world_religions" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/_mmlu.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/_mmlu.yaml deleted file mode 100644 index 98de9bfe..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/_mmlu.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: mmlu_alt_ov_01b -task: - - mmlu_alt_ov_01b_stem - - mmlu_alt_ov_01b_other - - mmlu_alt_ov_01b_social_sciences - - mmlu_alt_ov_01b_humanities diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/_template_yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/_template_yaml deleted file mode 100644 index 3fc9f640..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/_template_yaml +++ /dev/null @@ -1,11 +0,0 @@ -dataset_path: hails/mmlu_no_train -test_split: test -fewshot_split: dev -output_type: multiple_choice -doc_to_text: !function ../../../styles.template_01 -doc_to_choice: !function ../../../styles.choice_01b -doc_to_target: answer -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_abstract_algebra.yaml deleted file mode 100644 index c729cd5d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_abstract_algebra.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "abstract_algebra" -"description": "The following are multiple choice questions (with answers) about abstract\ - \ algebra.\n\n" -"group": "mmlu_alt_ov_01b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_anatomy.yaml deleted file mode 100644 index 1f0fdcc0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_anatomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "anatomy" -"description": "The following are multiple choice questions (with answers) about anatomy.\n\ - \n" -"group": "mmlu_alt_ov_01b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_anatomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_astronomy.yaml deleted file mode 100644 index 52479b69..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_astronomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "astronomy" -"description": "The following are multiple choice questions (with answers) about astronomy.\n\ - \n" -"group": "mmlu_alt_ov_01b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_astronomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_business_ethics.yaml deleted file mode 100644 index a6b2529c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_business_ethics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "business_ethics" -"description": "The following are multiple choice questions (with answers) about business\ - \ ethics.\n\n" -"group": "mmlu_alt_ov_01b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_business_ethics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_clinical_knowledge.yaml deleted file mode 100644 index 48d18e21..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_clinical_knowledge.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "clinical_knowledge" -"description": "The following are multiple choice questions (with answers) about clinical\ - \ knowledge.\n\n" -"group": "mmlu_alt_ov_01b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_college_biology.yaml deleted file mode 100644 index 67be6bde..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_college_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_biology" -"description": "The following are multiple choice questions (with answers) about college\ - \ biology.\n\n" -"group": "mmlu_alt_ov_01b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_college_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_college_chemistry.yaml deleted file mode 100644 index b5a2b281..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_college_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_chemistry" -"description": "The following are multiple choice questions (with answers) about college\ - \ chemistry.\n\n" -"group": "mmlu_alt_ov_01b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_college_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_college_computer_science.yaml deleted file mode 100644 index 8e27b86f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_college_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_computer_science" -"description": "The following are multiple choice questions (with answers) about college\ - \ computer science.\n\n" -"group": "mmlu_alt_ov_01b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_college_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_college_mathematics.yaml deleted file mode 100644 index adc7363e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_college_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_mathematics" -"description": "The following are multiple choice questions (with answers) about college\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_01b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_college_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_college_medicine.yaml deleted file mode 100644 index 10c82eca..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_college_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_medicine" -"description": "The following are multiple choice questions (with answers) about college\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_01b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_college_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_college_physics.yaml deleted file mode 100644 index af4b7ac1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_college_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_physics" -"description": "The following are multiple choice questions (with answers) about college\ - \ physics.\n\n" -"group": "mmlu_alt_ov_01b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_college_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_computer_security.yaml deleted file mode 100644 index e261a81f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_computer_security.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "computer_security" -"description": "The following are multiple choice questions (with answers) about computer\ - \ security.\n\n" -"group": "mmlu_alt_ov_01b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_computer_security" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_conceptual_physics.yaml deleted file mode 100644 index b18aff6b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_conceptual_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "conceptual_physics" -"description": "The following are multiple choice questions (with answers) about conceptual\ - \ physics.\n\n" -"group": "mmlu_alt_ov_01b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_econometrics.yaml deleted file mode 100644 index 8b27fcbc..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_econometrics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "econometrics" -"description": "The following are multiple choice questions (with answers) about econometrics.\n\ - \n" -"group": "mmlu_alt_ov_01b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_econometrics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_electrical_engineering.yaml deleted file mode 100644 index 5b6b77db..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_electrical_engineering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "electrical_engineering" -"description": "The following are multiple choice questions (with answers) about electrical\ - \ engineering.\n\n" -"group": "mmlu_alt_ov_01b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_elementary_mathematics.yaml deleted file mode 100644 index b388c142..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_elementary_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "elementary_mathematics" -"description": "The following are multiple choice questions (with answers) about elementary\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_01b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_formal_logic.yaml deleted file mode 100644 index 380b2c72..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_formal_logic.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_logic" -"description": "The following are multiple choice questions (with answers) about formal\ - \ logic.\n\n" -"group": "mmlu_alt_ov_01b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_formal_logic" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_global_facts.yaml deleted file mode 100644 index 666586e6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_global_facts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "global_facts" -"description": "The following are multiple choice questions (with answers) about global\ - \ facts.\n\n" -"group": "mmlu_alt_ov_01b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_global_facts" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_biology.yaml deleted file mode 100644 index 7c3c0284..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_biology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school biology.\n\n" -"group": "mmlu_alt_ov_01b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_high_school_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_chemistry.yaml deleted file mode 100644 index 7c456554..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_chemistry" -"description": "The following are multiple choice questions (with answers) about high\ - \ school chemistry.\n\n" -"group": "mmlu_alt_ov_01b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_computer_science.yaml deleted file mode 100644 index ce932c88..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_computer_science" -"description": "The following are multiple choice questions (with answers) about high\ - \ school computer science.\n\n" -"group": "mmlu_alt_ov_01b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_european_history.yaml deleted file mode 100644 index e74ac6fb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_european_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_european_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school european history.\n\n" -"group": "mmlu_alt_ov_01b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_geography.yaml deleted file mode 100644 index dad2dad4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_geography.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_geography" -"description": "The following are multiple choice questions (with answers) about high\ - \ school geography.\n\n" -"group": "mmlu_alt_ov_01b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_high_school_geography" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_government_and_politics.yaml deleted file mode 100644 index 2e646650..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_government_and_politics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_government_and_politics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school government and politics.\n\n" -"group": "mmlu_alt_ov_01b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_macroeconomics.yaml deleted file mode 100644 index 3b648898..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_macroeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_macroeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school macroeconomics.\n\n" -"group": "mmlu_alt_ov_01b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_mathematics.yaml deleted file mode 100644 index 2d396ff6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_mathematics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school mathematics.\n\n" -"group": "mmlu_alt_ov_01b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_microeconomics.yaml deleted file mode 100644 index ed595897..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_microeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_microeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school microeconomics.\n\n" -"group": "mmlu_alt_ov_01b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_physics.yaml deleted file mode 100644 index 9e15d293..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_physics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school physics.\n\n" -"group": "mmlu_alt_ov_01b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_high_school_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_psychology.yaml deleted file mode 100644 index c2ad6c7f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_psychology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school psychology.\n\n" -"group": "mmlu_alt_ov_01b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_statistics.yaml deleted file mode 100644 index f9e7e8e5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_statistics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_statistics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school statistics.\n\n" -"group": "mmlu_alt_ov_01b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_us_history.yaml deleted file mode 100644 index eb383e71..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_us_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_us_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school us history.\n\n" -"group": "mmlu_alt_ov_01b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_world_history.yaml deleted file mode 100644 index c5942ac0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_high_school_world_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_world_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school world history.\n\n" -"group": "mmlu_alt_ov_01b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_human_aging.yaml deleted file mode 100644 index 34b171b1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_human_aging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_aging" -"description": "The following are multiple choice questions (with answers) about human\ - \ aging.\n\n" -"group": "mmlu_alt_ov_01b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_human_aging" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_human_sexuality.yaml deleted file mode 100644 index 890dc96a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_human_sexuality.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_sexuality" -"description": "The following are multiple choice questions (with answers) about human\ - \ sexuality.\n\n" -"group": "mmlu_alt_ov_01b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_human_sexuality" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_international_law.yaml deleted file mode 100644 index 915a2e9f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_international_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "international_law" -"description": "The following are multiple choice questions (with answers) about international\ - \ law.\n\n" -"group": "mmlu_alt_ov_01b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_international_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_jurisprudence.yaml deleted file mode 100644 index 61824c8f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_jurisprudence.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "jurisprudence" -"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ - \n" -"group": "mmlu_alt_ov_01b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_jurisprudence" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_logical_fallacies.yaml deleted file mode 100644 index 393e1e74..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_logical_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "logical_fallacies" -"description": "The following are multiple choice questions (with answers) about logical\ - \ fallacies.\n\n" -"group": "mmlu_alt_ov_01b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_machine_learning.yaml deleted file mode 100644 index 12bd3d36..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_machine_learning.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "machine_learning" -"description": "The following are multiple choice questions (with answers) about machine\ - \ learning.\n\n" -"group": "mmlu_alt_ov_01b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_machine_learning" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_management.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_management.yaml deleted file mode 100644 index 36c9e6cc..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_management.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "management" -"description": "The following are multiple choice questions (with answers) about management.\n\ - \n" -"group": "mmlu_alt_ov_01b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_management" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_marketing.yaml deleted file mode 100644 index 8578bb86..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_marketing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "marketing" -"description": "The following are multiple choice questions (with answers) about marketing.\n\ - \n" -"group": "mmlu_alt_ov_01b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_marketing" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_medical_genetics.yaml deleted file mode 100644 index 55304f5b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_medical_genetics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "medical_genetics" -"description": "The following are multiple choice questions (with answers) about medical\ - \ genetics.\n\n" -"group": "mmlu_alt_ov_01b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_medical_genetics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_miscellaneous.yaml deleted file mode 100644 index 6fb4b8dd..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_miscellaneous.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "miscellaneous" -"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ - \n" -"group": "mmlu_alt_ov_01b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_miscellaneous" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_moral_disputes.yaml deleted file mode 100644 index 5bf805a7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_moral_disputes.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_disputes" -"description": "The following are multiple choice questions (with answers) about moral\ - \ disputes.\n\n" -"group": "mmlu_alt_ov_01b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_moral_disputes" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_moral_scenarios.yaml deleted file mode 100644 index 322a98d5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_moral_scenarios.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_scenarios" -"description": "The following are multiple choice questions (with answers) about moral\ - \ scenarios.\n\n" -"group": "mmlu_alt_ov_01b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_nutrition.yaml deleted file mode 100644 index 0f3553b3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_nutrition.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "nutrition" -"description": "The following are multiple choice questions (with answers) about nutrition.\n\ - \n" -"group": "mmlu_alt_ov_01b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_nutrition" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_philosophy.yaml deleted file mode 100644 index badf51ea..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_philosophy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "philosophy" -"description": "The following are multiple choice questions (with answers) about philosophy.\n\ - \n" -"group": "mmlu_alt_ov_01b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_philosophy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_prehistory.yaml deleted file mode 100644 index eec54ea0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_prehistory.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "prehistory" -"description": "The following are multiple choice questions (with answers) about prehistory.\n\ - \n" -"group": "mmlu_alt_ov_01b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_prehistory" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_professional_accounting.yaml deleted file mode 100644 index bed584f3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_professional_accounting.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_accounting" -"description": "The following are multiple choice questions (with answers) about professional\ - \ accounting.\n\n" -"group": "mmlu_alt_ov_01b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_professional_accounting" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_professional_law.yaml deleted file mode 100644 index 574f13ff..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_professional_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_law" -"description": "The following are multiple choice questions (with answers) about professional\ - \ law.\n\n" -"group": "mmlu_alt_ov_01b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_professional_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_professional_medicine.yaml deleted file mode 100644 index 3913e4b1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_professional_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_medicine" -"description": "The following are multiple choice questions (with answers) about professional\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_01b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_professional_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_professional_psychology.yaml deleted file mode 100644 index eb612a97..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_professional_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_psychology" -"description": "The following are multiple choice questions (with answers) about professional\ - \ psychology.\n\n" -"group": "mmlu_alt_ov_01b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_professional_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_public_relations.yaml deleted file mode 100644 index 36e179e8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_public_relations.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "public_relations" -"description": "The following are multiple choice questions (with answers) about public\ - \ relations.\n\n" -"group": "mmlu_alt_ov_01b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_public_relations" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_security_studies.yaml deleted file mode 100644 index c907ec26..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_security_studies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "security_studies" -"description": "The following are multiple choice questions (with answers) about security\ - \ studies.\n\n" -"group": "mmlu_alt_ov_01b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_security_studies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_sociology.yaml deleted file mode 100644 index 0045baee..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_sociology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sociology" -"description": "The following are multiple choice questions (with answers) about sociology.\n\ - \n" -"group": "mmlu_alt_ov_01b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_sociology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_us_foreign_policy.yaml deleted file mode 100644 index 5fa5b959..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_us_foreign_policy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "us_foreign_policy" -"description": "The following are multiple choice questions (with answers) about us\ - \ foreign policy.\n\n" -"group": "mmlu_alt_ov_01b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_virology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_virology.yaml deleted file mode 100644 index f045210b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_virology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "virology" -"description": "The following are multiple choice questions (with answers) about virology.\n\ - \n" -"group": "mmlu_alt_ov_01b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_virology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_world_religions.yaml deleted file mode 100644 index d252c730..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/b/mmlu_world_religions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "world_religions" -"description": "The following are multiple choice questions (with answers) about world\ - \ religions.\n\n" -"group": "mmlu_alt_ov_01b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01b_world_religions" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/_mmlu.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/_mmlu.yaml deleted file mode 100644 index e3a06f31..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/_mmlu.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: mmlu_alt_ov_01c -task: - - mmlu_alt_ov_01c_stem - - mmlu_alt_ov_01c_other - - mmlu_alt_ov_01c_social_sciences - - mmlu_alt_ov_01c_humanities diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/_template_yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/_template_yaml deleted file mode 100644 index aa68959a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/_template_yaml +++ /dev/null @@ -1,11 +0,0 @@ -dataset_path: hails/mmlu_no_train -test_split: test -fewshot_split: dev -output_type: multiple_choice -doc_to_text: !function ../../../styles.template_01 -doc_to_choice: !function ../../../styles.choice_01c -doc_to_target: answer -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_abstract_algebra.yaml deleted file mode 100644 index f39778db..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_abstract_algebra.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "abstract_algebra" -"description": "The following are multiple choice questions (with answers) about abstract\ - \ algebra.\n\n" -"group": "mmlu_alt_ov_01c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_anatomy.yaml deleted file mode 100644 index b3df09f5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_anatomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "anatomy" -"description": "The following are multiple choice questions (with answers) about anatomy.\n\ - \n" -"group": "mmlu_alt_ov_01c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_anatomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_astronomy.yaml deleted file mode 100644 index 2ab730aa..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_astronomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "astronomy" -"description": "The following are multiple choice questions (with answers) about astronomy.\n\ - \n" -"group": "mmlu_alt_ov_01c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_astronomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_business_ethics.yaml deleted file mode 100644 index aaf6bce4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_business_ethics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "business_ethics" -"description": "The following are multiple choice questions (with answers) about business\ - \ ethics.\n\n" -"group": "mmlu_alt_ov_01c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_business_ethics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_clinical_knowledge.yaml deleted file mode 100644 index 26c029b9..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_clinical_knowledge.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "clinical_knowledge" -"description": "The following are multiple choice questions (with answers) about clinical\ - \ knowledge.\n\n" -"group": "mmlu_alt_ov_01c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_college_biology.yaml deleted file mode 100644 index e7ed41b3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_college_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_biology" -"description": "The following are multiple choice questions (with answers) about college\ - \ biology.\n\n" -"group": "mmlu_alt_ov_01c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_college_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_college_chemistry.yaml deleted file mode 100644 index d3c11799..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_college_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_chemistry" -"description": "The following are multiple choice questions (with answers) about college\ - \ chemistry.\n\n" -"group": "mmlu_alt_ov_01c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_college_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_college_computer_science.yaml deleted file mode 100644 index 65b4db34..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_college_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_computer_science" -"description": "The following are multiple choice questions (with answers) about college\ - \ computer science.\n\n" -"group": "mmlu_alt_ov_01c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_college_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_college_mathematics.yaml deleted file mode 100644 index eaa9f427..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_college_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_mathematics" -"description": "The following are multiple choice questions (with answers) about college\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_01c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_college_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_college_medicine.yaml deleted file mode 100644 index 6e4c6877..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_college_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_medicine" -"description": "The following are multiple choice questions (with answers) about college\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_01c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_college_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_college_physics.yaml deleted file mode 100644 index 0f30cc94..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_college_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_physics" -"description": "The following are multiple choice questions (with answers) about college\ - \ physics.\n\n" -"group": "mmlu_alt_ov_01c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_college_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_computer_security.yaml deleted file mode 100644 index 4ba1c770..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_computer_security.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "computer_security" -"description": "The following are multiple choice questions (with answers) about computer\ - \ security.\n\n" -"group": "mmlu_alt_ov_01c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_computer_security" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_conceptual_physics.yaml deleted file mode 100644 index 1a0fac6d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_conceptual_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "conceptual_physics" -"description": "The following are multiple choice questions (with answers) about conceptual\ - \ physics.\n\n" -"group": "mmlu_alt_ov_01c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_econometrics.yaml deleted file mode 100644 index 16953f87..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_econometrics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "econometrics" -"description": "The following are multiple choice questions (with answers) about econometrics.\n\ - \n" -"group": "mmlu_alt_ov_01c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_econometrics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_electrical_engineering.yaml deleted file mode 100644 index a24deb27..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_electrical_engineering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "electrical_engineering" -"description": "The following are multiple choice questions (with answers) about electrical\ - \ engineering.\n\n" -"group": "mmlu_alt_ov_01c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_elementary_mathematics.yaml deleted file mode 100644 index 78f00f57..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_elementary_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "elementary_mathematics" -"description": "The following are multiple choice questions (with answers) about elementary\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_01c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_formal_logic.yaml deleted file mode 100644 index b96a0780..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_formal_logic.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_logic" -"description": "The following are multiple choice questions (with answers) about formal\ - \ logic.\n\n" -"group": "mmlu_alt_ov_01c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_formal_logic" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_global_facts.yaml deleted file mode 100644 index 06c6be19..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_global_facts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "global_facts" -"description": "The following are multiple choice questions (with answers) about global\ - \ facts.\n\n" -"group": "mmlu_alt_ov_01c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_global_facts" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_biology.yaml deleted file mode 100644 index 5a54c7e5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_biology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school biology.\n\n" -"group": "mmlu_alt_ov_01c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_high_school_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_chemistry.yaml deleted file mode 100644 index 8e30a672..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_chemistry" -"description": "The following are multiple choice questions (with answers) about high\ - \ school chemistry.\n\n" -"group": "mmlu_alt_ov_01c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_computer_science.yaml deleted file mode 100644 index 2af74be5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_computer_science" -"description": "The following are multiple choice questions (with answers) about high\ - \ school computer science.\n\n" -"group": "mmlu_alt_ov_01c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_european_history.yaml deleted file mode 100644 index 268121e5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_european_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_european_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school european history.\n\n" -"group": "mmlu_alt_ov_01c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_geography.yaml deleted file mode 100644 index 120e737f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_geography.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_geography" -"description": "The following are multiple choice questions (with answers) about high\ - \ school geography.\n\n" -"group": "mmlu_alt_ov_01c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_high_school_geography" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_government_and_politics.yaml deleted file mode 100644 index db0fa59d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_government_and_politics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_government_and_politics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school government and politics.\n\n" -"group": "mmlu_alt_ov_01c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_macroeconomics.yaml deleted file mode 100644 index 979ff8b7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_macroeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_macroeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school macroeconomics.\n\n" -"group": "mmlu_alt_ov_01c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_mathematics.yaml deleted file mode 100644 index 186799f0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_mathematics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school mathematics.\n\n" -"group": "mmlu_alt_ov_01c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_microeconomics.yaml deleted file mode 100644 index 1b6fcc67..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_microeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_microeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school microeconomics.\n\n" -"group": "mmlu_alt_ov_01c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_physics.yaml deleted file mode 100644 index d733408b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_physics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school physics.\n\n" -"group": "mmlu_alt_ov_01c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_high_school_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_psychology.yaml deleted file mode 100644 index 0005c6cd..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_psychology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school psychology.\n\n" -"group": "mmlu_alt_ov_01c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_statistics.yaml deleted file mode 100644 index 262a2ae4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_statistics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_statistics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school statistics.\n\n" -"group": "mmlu_alt_ov_01c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_us_history.yaml deleted file mode 100644 index 53d3bb67..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_us_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_us_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school us history.\n\n" -"group": "mmlu_alt_ov_01c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_world_history.yaml deleted file mode 100644 index ccf074ae..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_high_school_world_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_world_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school world history.\n\n" -"group": "mmlu_alt_ov_01c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_human_aging.yaml deleted file mode 100644 index 5365955d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_human_aging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_aging" -"description": "The following are multiple choice questions (with answers) about human\ - \ aging.\n\n" -"group": "mmlu_alt_ov_01c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_human_aging" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_human_sexuality.yaml deleted file mode 100644 index fb2c8df3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_human_sexuality.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_sexuality" -"description": "The following are multiple choice questions (with answers) about human\ - \ sexuality.\n\n" -"group": "mmlu_alt_ov_01c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_human_sexuality" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_international_law.yaml deleted file mode 100644 index a8d94795..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_international_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "international_law" -"description": "The following are multiple choice questions (with answers) about international\ - \ law.\n\n" -"group": "mmlu_alt_ov_01c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_international_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_jurisprudence.yaml deleted file mode 100644 index d2eebff2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_jurisprudence.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "jurisprudence" -"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ - \n" -"group": "mmlu_alt_ov_01c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_jurisprudence" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_logical_fallacies.yaml deleted file mode 100644 index 20e3d93f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_logical_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "logical_fallacies" -"description": "The following are multiple choice questions (with answers) about logical\ - \ fallacies.\n\n" -"group": "mmlu_alt_ov_01c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_machine_learning.yaml deleted file mode 100644 index c2af3d02..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_machine_learning.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "machine_learning" -"description": "The following are multiple choice questions (with answers) about machine\ - \ learning.\n\n" -"group": "mmlu_alt_ov_01c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_machine_learning" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_management.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_management.yaml deleted file mode 100644 index 44d477cb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_management.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "management" -"description": "The following are multiple choice questions (with answers) about management.\n\ - \n" -"group": "mmlu_alt_ov_01c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_management" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_marketing.yaml deleted file mode 100644 index 424f941e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_marketing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "marketing" -"description": "The following are multiple choice questions (with answers) about marketing.\n\ - \n" -"group": "mmlu_alt_ov_01c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_marketing" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_medical_genetics.yaml deleted file mode 100644 index 540aba6b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_medical_genetics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "medical_genetics" -"description": "The following are multiple choice questions (with answers) about medical\ - \ genetics.\n\n" -"group": "mmlu_alt_ov_01c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_medical_genetics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_miscellaneous.yaml deleted file mode 100644 index d76c9116..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_miscellaneous.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "miscellaneous" -"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ - \n" -"group": "mmlu_alt_ov_01c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_miscellaneous" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_moral_disputes.yaml deleted file mode 100644 index d3674820..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_moral_disputes.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_disputes" -"description": "The following are multiple choice questions (with answers) about moral\ - \ disputes.\n\n" -"group": "mmlu_alt_ov_01c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_moral_disputes" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_moral_scenarios.yaml deleted file mode 100644 index eeb089c8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_moral_scenarios.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_scenarios" -"description": "The following are multiple choice questions (with answers) about moral\ - \ scenarios.\n\n" -"group": "mmlu_alt_ov_01c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_nutrition.yaml deleted file mode 100644 index ed8cd5bb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_nutrition.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "nutrition" -"description": "The following are multiple choice questions (with answers) about nutrition.\n\ - \n" -"group": "mmlu_alt_ov_01c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_nutrition" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_philosophy.yaml deleted file mode 100644 index c40e4600..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_philosophy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "philosophy" -"description": "The following are multiple choice questions (with answers) about philosophy.\n\ - \n" -"group": "mmlu_alt_ov_01c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_philosophy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_prehistory.yaml deleted file mode 100644 index 9e48d0a5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_prehistory.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "prehistory" -"description": "The following are multiple choice questions (with answers) about prehistory.\n\ - \n" -"group": "mmlu_alt_ov_01c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_prehistory" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_professional_accounting.yaml deleted file mode 100644 index 89660aae..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_professional_accounting.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_accounting" -"description": "The following are multiple choice questions (with answers) about professional\ - \ accounting.\n\n" -"group": "mmlu_alt_ov_01c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_professional_accounting" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_professional_law.yaml deleted file mode 100644 index 4fa85794..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_professional_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_law" -"description": "The following are multiple choice questions (with answers) about professional\ - \ law.\n\n" -"group": "mmlu_alt_ov_01c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_professional_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_professional_medicine.yaml deleted file mode 100644 index a59797c1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_professional_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_medicine" -"description": "The following are multiple choice questions (with answers) about professional\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_01c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_professional_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_professional_psychology.yaml deleted file mode 100644 index b7270adb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_professional_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_psychology" -"description": "The following are multiple choice questions (with answers) about professional\ - \ psychology.\n\n" -"group": "mmlu_alt_ov_01c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_professional_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_public_relations.yaml deleted file mode 100644 index 5e830d81..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_public_relations.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "public_relations" -"description": "The following are multiple choice questions (with answers) about public\ - \ relations.\n\n" -"group": "mmlu_alt_ov_01c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_public_relations" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_security_studies.yaml deleted file mode 100644 index 13d8a3ec..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_security_studies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "security_studies" -"description": "The following are multiple choice questions (with answers) about security\ - \ studies.\n\n" -"group": "mmlu_alt_ov_01c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_security_studies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_sociology.yaml deleted file mode 100644 index 20259753..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_sociology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sociology" -"description": "The following are multiple choice questions (with answers) about sociology.\n\ - \n" -"group": "mmlu_alt_ov_01c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_sociology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_us_foreign_policy.yaml deleted file mode 100644 index ef314a4f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_us_foreign_policy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "us_foreign_policy" -"description": "The following are multiple choice questions (with answers) about us\ - \ foreign policy.\n\n" -"group": "mmlu_alt_ov_01c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_virology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_virology.yaml deleted file mode 100644 index 8f4d3c87..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_virology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "virology" -"description": "The following are multiple choice questions (with answers) about virology.\n\ - \n" -"group": "mmlu_alt_ov_01c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_virology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_world_religions.yaml deleted file mode 100644 index a5e54260..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/c/mmlu_world_religions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "world_religions" -"description": "The following are multiple choice questions (with answers) about world\ - \ religions.\n\n" -"group": "mmlu_alt_ov_01c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_01c_world_religions" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/_mmlu_ov_02.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/_mmlu_ov_02.yaml deleted file mode 100644 index b7bb3acc..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/_mmlu_ov_02.yaml +++ /dev/null @@ -1,5 +0,0 @@ -group: mmlu_alt_ov_02 -task: - - mmlu_alt_ov_02a - - mmlu_alt_ov_02b - - mmlu_alt_ov_02c diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/_mmlu.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/_mmlu.yaml deleted file mode 100644 index fb86cab1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/_mmlu.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: mmlu_alt_ov_02a -task: - - mmlu_alt_ov_02a_stem - - mmlu_alt_ov_02a_other - - mmlu_alt_ov_02a_social_sciences - - mmlu_alt_ov_02a_humanities diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/_template_yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/_template_yaml deleted file mode 100644 index f9d0cbe2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/_template_yaml +++ /dev/null @@ -1,11 +0,0 @@ -dataset_path: hails/mmlu_no_train -test_split: test -fewshot_split: dev -output_type: multiple_choice -doc_to_text: !function ../../../styles.template_02 -doc_to_choice: !function ../../../styles.choice_02a -doc_to_target: answer -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_abstract_algebra.yaml deleted file mode 100644 index 48affab8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_abstract_algebra.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "abstract_algebra" -"description": "The following are multiple choice questions (with answers) about abstract\ - \ algebra.\n\n" -"group": "mmlu_alt_ov_02a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_anatomy.yaml deleted file mode 100644 index 22c431f2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_anatomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "anatomy" -"description": "The following are multiple choice questions (with answers) about anatomy.\n\ - \n" -"group": "mmlu_alt_ov_02a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_anatomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_astronomy.yaml deleted file mode 100644 index 9d2c6ef8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_astronomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "astronomy" -"description": "The following are multiple choice questions (with answers) about astronomy.\n\ - \n" -"group": "mmlu_alt_ov_02a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_astronomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_business_ethics.yaml deleted file mode 100644 index 909cacad..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_business_ethics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "business_ethics" -"description": "The following are multiple choice questions (with answers) about business\ - \ ethics.\n\n" -"group": "mmlu_alt_ov_02a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_business_ethics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_clinical_knowledge.yaml deleted file mode 100644 index f954bef2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_clinical_knowledge.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "clinical_knowledge" -"description": "The following are multiple choice questions (with answers) about clinical\ - \ knowledge.\n\n" -"group": "mmlu_alt_ov_02a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_college_biology.yaml deleted file mode 100644 index a7f3b503..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_college_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_biology" -"description": "The following are multiple choice questions (with answers) about college\ - \ biology.\n\n" -"group": "mmlu_alt_ov_02a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_college_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_college_chemistry.yaml deleted file mode 100644 index 43f59c40..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_college_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_chemistry" -"description": "The following are multiple choice questions (with answers) about college\ - \ chemistry.\n\n" -"group": "mmlu_alt_ov_02a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_college_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_college_computer_science.yaml deleted file mode 100644 index 6e9e50b4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_college_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_computer_science" -"description": "The following are multiple choice questions (with answers) about college\ - \ computer science.\n\n" -"group": "mmlu_alt_ov_02a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_college_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_college_mathematics.yaml deleted file mode 100644 index 4fa13ec2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_college_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_mathematics" -"description": "The following are multiple choice questions (with answers) about college\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_02a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_college_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_college_medicine.yaml deleted file mode 100644 index 8dd5e59d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_college_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_medicine" -"description": "The following are multiple choice questions (with answers) about college\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_02a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_college_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_college_physics.yaml deleted file mode 100644 index b5d48026..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_college_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_physics" -"description": "The following are multiple choice questions (with answers) about college\ - \ physics.\n\n" -"group": "mmlu_alt_ov_02a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_college_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_computer_security.yaml deleted file mode 100644 index c51793c2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_computer_security.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "computer_security" -"description": "The following are multiple choice questions (with answers) about computer\ - \ security.\n\n" -"group": "mmlu_alt_ov_02a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_computer_security" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_conceptual_physics.yaml deleted file mode 100644 index 0ae991c7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_conceptual_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "conceptual_physics" -"description": "The following are multiple choice questions (with answers) about conceptual\ - \ physics.\n\n" -"group": "mmlu_alt_ov_02a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_econometrics.yaml deleted file mode 100644 index 9c2453b3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_econometrics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "econometrics" -"description": "The following are multiple choice questions (with answers) about econometrics.\n\ - \n" -"group": "mmlu_alt_ov_02a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_econometrics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_electrical_engineering.yaml deleted file mode 100644 index f84ea46b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_electrical_engineering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "electrical_engineering" -"description": "The following are multiple choice questions (with answers) about electrical\ - \ engineering.\n\n" -"group": "mmlu_alt_ov_02a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_elementary_mathematics.yaml deleted file mode 100644 index 70650738..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_elementary_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "elementary_mathematics" -"description": "The following are multiple choice questions (with answers) about elementary\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_02a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_formal_logic.yaml deleted file mode 100644 index 58bed110..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_formal_logic.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_logic" -"description": "The following are multiple choice questions (with answers) about formal\ - \ logic.\n\n" -"group": "mmlu_alt_ov_02a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_formal_logic" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_global_facts.yaml deleted file mode 100644 index 67c2bccb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_global_facts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "global_facts" -"description": "The following are multiple choice questions (with answers) about global\ - \ facts.\n\n" -"group": "mmlu_alt_ov_02a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_global_facts" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_biology.yaml deleted file mode 100644 index 606cf7cb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_biology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school biology.\n\n" -"group": "mmlu_alt_ov_02a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_high_school_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_chemistry.yaml deleted file mode 100644 index a263a8db..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_chemistry" -"description": "The following are multiple choice questions (with answers) about high\ - \ school chemistry.\n\n" -"group": "mmlu_alt_ov_02a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_computer_science.yaml deleted file mode 100644 index aaf11f19..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_computer_science" -"description": "The following are multiple choice questions (with answers) about high\ - \ school computer science.\n\n" -"group": "mmlu_alt_ov_02a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_european_history.yaml deleted file mode 100644 index 16d3c792..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_european_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_european_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school european history.\n\n" -"group": "mmlu_alt_ov_02a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_geography.yaml deleted file mode 100644 index 2890a9ca..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_geography.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_geography" -"description": "The following are multiple choice questions (with answers) about high\ - \ school geography.\n\n" -"group": "mmlu_alt_ov_02a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_high_school_geography" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_government_and_politics.yaml deleted file mode 100644 index 7f9ea1ff..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_government_and_politics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_government_and_politics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school government and politics.\n\n" -"group": "mmlu_alt_ov_02a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_macroeconomics.yaml deleted file mode 100644 index 3f9d9bc0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_macroeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_macroeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school macroeconomics.\n\n" -"group": "mmlu_alt_ov_02a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_mathematics.yaml deleted file mode 100644 index 95849a8a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_mathematics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school mathematics.\n\n" -"group": "mmlu_alt_ov_02a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_microeconomics.yaml deleted file mode 100644 index 4a47d2b3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_microeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_microeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school microeconomics.\n\n" -"group": "mmlu_alt_ov_02a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_physics.yaml deleted file mode 100644 index d4bc7e85..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_physics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school physics.\n\n" -"group": "mmlu_alt_ov_02a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_high_school_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_psychology.yaml deleted file mode 100644 index a68cad24..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_psychology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school psychology.\n\n" -"group": "mmlu_alt_ov_02a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_statistics.yaml deleted file mode 100644 index cc69fe7e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_statistics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_statistics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school statistics.\n\n" -"group": "mmlu_alt_ov_02a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_us_history.yaml deleted file mode 100644 index f2d93fd2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_us_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_us_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school us history.\n\n" -"group": "mmlu_alt_ov_02a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_world_history.yaml deleted file mode 100644 index 09cca52d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_high_school_world_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_world_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school world history.\n\n" -"group": "mmlu_alt_ov_02a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_human_aging.yaml deleted file mode 100644 index 6e13b1cc..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_human_aging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_aging" -"description": "The following are multiple choice questions (with answers) about human\ - \ aging.\n\n" -"group": "mmlu_alt_ov_02a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_human_aging" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_human_sexuality.yaml deleted file mode 100644 index e41cde9f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_human_sexuality.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_sexuality" -"description": "The following are multiple choice questions (with answers) about human\ - \ sexuality.\n\n" -"group": "mmlu_alt_ov_02a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_human_sexuality" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_international_law.yaml deleted file mode 100644 index e7329d83..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_international_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "international_law" -"description": "The following are multiple choice questions (with answers) about international\ - \ law.\n\n" -"group": "mmlu_alt_ov_02a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_international_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_jurisprudence.yaml deleted file mode 100644 index 83e9ba83..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_jurisprudence.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "jurisprudence" -"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ - \n" -"group": "mmlu_alt_ov_02a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_jurisprudence" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_logical_fallacies.yaml deleted file mode 100644 index 19f7be70..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_logical_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "logical_fallacies" -"description": "The following are multiple choice questions (with answers) about logical\ - \ fallacies.\n\n" -"group": "mmlu_alt_ov_02a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_machine_learning.yaml deleted file mode 100644 index fd9c96ee..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_machine_learning.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "machine_learning" -"description": "The following are multiple choice questions (with answers) about machine\ - \ learning.\n\n" -"group": "mmlu_alt_ov_02a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_machine_learning" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_management.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_management.yaml deleted file mode 100644 index f57c2349..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_management.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "management" -"description": "The following are multiple choice questions (with answers) about management.\n\ - \n" -"group": "mmlu_alt_ov_02a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_management" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_marketing.yaml deleted file mode 100644 index 96eae4ad..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_marketing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "marketing" -"description": "The following are multiple choice questions (with answers) about marketing.\n\ - \n" -"group": "mmlu_alt_ov_02a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_marketing" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_medical_genetics.yaml deleted file mode 100644 index be684a67..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_medical_genetics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "medical_genetics" -"description": "The following are multiple choice questions (with answers) about medical\ - \ genetics.\n\n" -"group": "mmlu_alt_ov_02a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_medical_genetics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_miscellaneous.yaml deleted file mode 100644 index 1987aa8f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_miscellaneous.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "miscellaneous" -"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ - \n" -"group": "mmlu_alt_ov_02a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_miscellaneous" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_moral_disputes.yaml deleted file mode 100644 index e0a677bd..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_moral_disputes.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_disputes" -"description": "The following are multiple choice questions (with answers) about moral\ - \ disputes.\n\n" -"group": "mmlu_alt_ov_02a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_moral_disputes" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_moral_scenarios.yaml deleted file mode 100644 index 5c0bfcf8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_moral_scenarios.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_scenarios" -"description": "The following are multiple choice questions (with answers) about moral\ - \ scenarios.\n\n" -"group": "mmlu_alt_ov_02a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_nutrition.yaml deleted file mode 100644 index 0a9a9b8f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_nutrition.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "nutrition" -"description": "The following are multiple choice questions (with answers) about nutrition.\n\ - \n" -"group": "mmlu_alt_ov_02a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_nutrition" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_philosophy.yaml deleted file mode 100644 index 3f8a63e4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_philosophy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "philosophy" -"description": "The following are multiple choice questions (with answers) about philosophy.\n\ - \n" -"group": "mmlu_alt_ov_02a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_philosophy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_prehistory.yaml deleted file mode 100644 index 5d0e79d2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_prehistory.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "prehistory" -"description": "The following are multiple choice questions (with answers) about prehistory.\n\ - \n" -"group": "mmlu_alt_ov_02a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_prehistory" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_professional_accounting.yaml deleted file mode 100644 index c32eb1f0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_professional_accounting.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_accounting" -"description": "The following are multiple choice questions (with answers) about professional\ - \ accounting.\n\n" -"group": "mmlu_alt_ov_02a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_professional_accounting" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_professional_law.yaml deleted file mode 100644 index 670486a0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_professional_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_law" -"description": "The following are multiple choice questions (with answers) about professional\ - \ law.\n\n" -"group": "mmlu_alt_ov_02a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_professional_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_professional_medicine.yaml deleted file mode 100644 index d2d8c0e1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_professional_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_medicine" -"description": "The following are multiple choice questions (with answers) about professional\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_02a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_professional_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_professional_psychology.yaml deleted file mode 100644 index 4539eb83..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_professional_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_psychology" -"description": "The following are multiple choice questions (with answers) about professional\ - \ psychology.\n\n" -"group": "mmlu_alt_ov_02a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_professional_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_public_relations.yaml deleted file mode 100644 index 9a91416a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_public_relations.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "public_relations" -"description": "The following are multiple choice questions (with answers) about public\ - \ relations.\n\n" -"group": "mmlu_alt_ov_02a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_public_relations" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_security_studies.yaml deleted file mode 100644 index 4563db67..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_security_studies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "security_studies" -"description": "The following are multiple choice questions (with answers) about security\ - \ studies.\n\n" -"group": "mmlu_alt_ov_02a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_security_studies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_sociology.yaml deleted file mode 100644 index 15a1c3cc..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_sociology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sociology" -"description": "The following are multiple choice questions (with answers) about sociology.\n\ - \n" -"group": "mmlu_alt_ov_02a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_sociology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_us_foreign_policy.yaml deleted file mode 100644 index 78b0a872..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_us_foreign_policy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "us_foreign_policy" -"description": "The following are multiple choice questions (with answers) about us\ - \ foreign policy.\n\n" -"group": "mmlu_alt_ov_02a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_virology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_virology.yaml deleted file mode 100644 index 5f2a94a9..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_virology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "virology" -"description": "The following are multiple choice questions (with answers) about virology.\n\ - \n" -"group": "mmlu_alt_ov_02a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_virology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_world_religions.yaml deleted file mode 100644 index 43680f9d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/a/mmlu_world_religions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "world_religions" -"description": "The following are multiple choice questions (with answers) about world\ - \ religions.\n\n" -"group": "mmlu_alt_ov_02a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02a_world_religions" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/_mmlu.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/_mmlu.yaml deleted file mode 100644 index e44e24d8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/_mmlu.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: mmlu_alt_ov_02b -task: - - mmlu_alt_ov_02b_stem - - mmlu_alt_ov_02b_other - - mmlu_alt_ov_02b_social_sciences - - mmlu_alt_ov_02b_humanities diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/_template_yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/_template_yaml deleted file mode 100644 index 61d42dc1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/_template_yaml +++ /dev/null @@ -1,11 +0,0 @@ -dataset_path: hails/mmlu_no_train -test_split: test -fewshot_split: dev -output_type: multiple_choice -doc_to_text: !function ../../../styles.template_02 -doc_to_choice: !function ../../../styles.choice_02b -doc_to_target: answer -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_abstract_algebra.yaml deleted file mode 100644 index beb0b444..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_abstract_algebra.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "abstract_algebra" -"description": "The following are multiple choice questions (with answers) about abstract\ - \ algebra.\n\n" -"group": "mmlu_alt_ov_02b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_anatomy.yaml deleted file mode 100644 index c191de4d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_anatomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "anatomy" -"description": "The following are multiple choice questions (with answers) about anatomy.\n\ - \n" -"group": "mmlu_alt_ov_02b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_anatomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_astronomy.yaml deleted file mode 100644 index dead2a72..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_astronomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "astronomy" -"description": "The following are multiple choice questions (with answers) about astronomy.\n\ - \n" -"group": "mmlu_alt_ov_02b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_astronomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_business_ethics.yaml deleted file mode 100644 index 7606d9de..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_business_ethics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "business_ethics" -"description": "The following are multiple choice questions (with answers) about business\ - \ ethics.\n\n" -"group": "mmlu_alt_ov_02b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_business_ethics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_clinical_knowledge.yaml deleted file mode 100644 index 590cc131..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_clinical_knowledge.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "clinical_knowledge" -"description": "The following are multiple choice questions (with answers) about clinical\ - \ knowledge.\n\n" -"group": "mmlu_alt_ov_02b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_college_biology.yaml deleted file mode 100644 index 70fb2a00..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_college_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_biology" -"description": "The following are multiple choice questions (with answers) about college\ - \ biology.\n\n" -"group": "mmlu_alt_ov_02b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_college_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_college_chemistry.yaml deleted file mode 100644 index d2507d3c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_college_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_chemistry" -"description": "The following are multiple choice questions (with answers) about college\ - \ chemistry.\n\n" -"group": "mmlu_alt_ov_02b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_college_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_college_computer_science.yaml deleted file mode 100644 index aa2d3135..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_college_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_computer_science" -"description": "The following are multiple choice questions (with answers) about college\ - \ computer science.\n\n" -"group": "mmlu_alt_ov_02b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_college_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_college_mathematics.yaml deleted file mode 100644 index b3b3df6d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_college_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_mathematics" -"description": "The following are multiple choice questions (with answers) about college\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_02b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_college_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_college_medicine.yaml deleted file mode 100644 index c0c13556..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_college_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_medicine" -"description": "The following are multiple choice questions (with answers) about college\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_02b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_college_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_college_physics.yaml deleted file mode 100644 index e6091f9f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_college_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_physics" -"description": "The following are multiple choice questions (with answers) about college\ - \ physics.\n\n" -"group": "mmlu_alt_ov_02b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_college_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_computer_security.yaml deleted file mode 100644 index cce99537..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_computer_security.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "computer_security" -"description": "The following are multiple choice questions (with answers) about computer\ - \ security.\n\n" -"group": "mmlu_alt_ov_02b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_computer_security" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_conceptual_physics.yaml deleted file mode 100644 index dd07cd4d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_conceptual_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "conceptual_physics" -"description": "The following are multiple choice questions (with answers) about conceptual\ - \ physics.\n\n" -"group": "mmlu_alt_ov_02b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_econometrics.yaml deleted file mode 100644 index 55fdada8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_econometrics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "econometrics" -"description": "The following are multiple choice questions (with answers) about econometrics.\n\ - \n" -"group": "mmlu_alt_ov_02b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_econometrics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_electrical_engineering.yaml deleted file mode 100644 index 23b5eca7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_electrical_engineering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "electrical_engineering" -"description": "The following are multiple choice questions (with answers) about electrical\ - \ engineering.\n\n" -"group": "mmlu_alt_ov_02b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_elementary_mathematics.yaml deleted file mode 100644 index 2968ce23..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_elementary_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "elementary_mathematics" -"description": "The following are multiple choice questions (with answers) about elementary\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_02b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_formal_logic.yaml deleted file mode 100644 index eb5607a8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_formal_logic.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_logic" -"description": "The following are multiple choice questions (with answers) about formal\ - \ logic.\n\n" -"group": "mmlu_alt_ov_02b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_formal_logic" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_global_facts.yaml deleted file mode 100644 index 23d91ba6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_global_facts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "global_facts" -"description": "The following are multiple choice questions (with answers) about global\ - \ facts.\n\n" -"group": "mmlu_alt_ov_02b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_global_facts" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_biology.yaml deleted file mode 100644 index 9a466aee..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_biology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school biology.\n\n" -"group": "mmlu_alt_ov_02b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_high_school_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_chemistry.yaml deleted file mode 100644 index ca1deecd..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_chemistry" -"description": "The following are multiple choice questions (with answers) about high\ - \ school chemistry.\n\n" -"group": "mmlu_alt_ov_02b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_computer_science.yaml deleted file mode 100644 index 8ac73230..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_computer_science" -"description": "The following are multiple choice questions (with answers) about high\ - \ school computer science.\n\n" -"group": "mmlu_alt_ov_02b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_european_history.yaml deleted file mode 100644 index 16e61fcc..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_european_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_european_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school european history.\n\n" -"group": "mmlu_alt_ov_02b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_geography.yaml deleted file mode 100644 index edc8d86a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_geography.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_geography" -"description": "The following are multiple choice questions (with answers) about high\ - \ school geography.\n\n" -"group": "mmlu_alt_ov_02b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_high_school_geography" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_government_and_politics.yaml deleted file mode 100644 index 5ee9b96f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_government_and_politics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_government_and_politics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school government and politics.\n\n" -"group": "mmlu_alt_ov_02b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_macroeconomics.yaml deleted file mode 100644 index b60e7ffd..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_macroeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_macroeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school macroeconomics.\n\n" -"group": "mmlu_alt_ov_02b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_mathematics.yaml deleted file mode 100644 index 232fd0dd..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_mathematics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school mathematics.\n\n" -"group": "mmlu_alt_ov_02b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_microeconomics.yaml deleted file mode 100644 index 2aff0816..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_microeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_microeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school microeconomics.\n\n" -"group": "mmlu_alt_ov_02b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_physics.yaml deleted file mode 100644 index 279ff330..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_physics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school physics.\n\n" -"group": "mmlu_alt_ov_02b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_high_school_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_psychology.yaml deleted file mode 100644 index 1045aae0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_psychology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school psychology.\n\n" -"group": "mmlu_alt_ov_02b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_statistics.yaml deleted file mode 100644 index d1aa0bd8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_statistics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_statistics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school statistics.\n\n" -"group": "mmlu_alt_ov_02b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_us_history.yaml deleted file mode 100644 index bd244d70..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_us_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_us_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school us history.\n\n" -"group": "mmlu_alt_ov_02b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_world_history.yaml deleted file mode 100644 index aef60150..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_high_school_world_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_world_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school world history.\n\n" -"group": "mmlu_alt_ov_02b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_human_aging.yaml deleted file mode 100644 index df776001..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_human_aging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_aging" -"description": "The following are multiple choice questions (with answers) about human\ - \ aging.\n\n" -"group": "mmlu_alt_ov_02b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_human_aging" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_human_sexuality.yaml deleted file mode 100644 index aff4c80a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_human_sexuality.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_sexuality" -"description": "The following are multiple choice questions (with answers) about human\ - \ sexuality.\n\n" -"group": "mmlu_alt_ov_02b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_human_sexuality" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_international_law.yaml deleted file mode 100644 index 86af3288..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_international_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "international_law" -"description": "The following are multiple choice questions (with answers) about international\ - \ law.\n\n" -"group": "mmlu_alt_ov_02b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_international_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_jurisprudence.yaml deleted file mode 100644 index 934f6dc2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_jurisprudence.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "jurisprudence" -"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ - \n" -"group": "mmlu_alt_ov_02b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_jurisprudence" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_logical_fallacies.yaml deleted file mode 100644 index 650cdb8e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_logical_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "logical_fallacies" -"description": "The following are multiple choice questions (with answers) about logical\ - \ fallacies.\n\n" -"group": "mmlu_alt_ov_02b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_machine_learning.yaml deleted file mode 100644 index ab6e4b89..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_machine_learning.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "machine_learning" -"description": "The following are multiple choice questions (with answers) about machine\ - \ learning.\n\n" -"group": "mmlu_alt_ov_02b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_machine_learning" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_management.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_management.yaml deleted file mode 100644 index 5172803a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_management.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "management" -"description": "The following are multiple choice questions (with answers) about management.\n\ - \n" -"group": "mmlu_alt_ov_02b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_management" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_marketing.yaml deleted file mode 100644 index a6a29fc1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_marketing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "marketing" -"description": "The following are multiple choice questions (with answers) about marketing.\n\ - \n" -"group": "mmlu_alt_ov_02b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_marketing" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_medical_genetics.yaml deleted file mode 100644 index 8f719609..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_medical_genetics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "medical_genetics" -"description": "The following are multiple choice questions (with answers) about medical\ - \ genetics.\n\n" -"group": "mmlu_alt_ov_02b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_medical_genetics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_miscellaneous.yaml deleted file mode 100644 index 74111ccf..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_miscellaneous.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "miscellaneous" -"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ - \n" -"group": "mmlu_alt_ov_02b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_miscellaneous" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_moral_disputes.yaml deleted file mode 100644 index 6ab4c663..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_moral_disputes.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_disputes" -"description": "The following are multiple choice questions (with answers) about moral\ - \ disputes.\n\n" -"group": "mmlu_alt_ov_02b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_moral_disputes" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_moral_scenarios.yaml deleted file mode 100644 index e3b1e389..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_moral_scenarios.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_scenarios" -"description": "The following are multiple choice questions (with answers) about moral\ - \ scenarios.\n\n" -"group": "mmlu_alt_ov_02b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_nutrition.yaml deleted file mode 100644 index a315c4a9..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_nutrition.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "nutrition" -"description": "The following are multiple choice questions (with answers) about nutrition.\n\ - \n" -"group": "mmlu_alt_ov_02b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_nutrition" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_philosophy.yaml deleted file mode 100644 index 613c9029..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_philosophy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "philosophy" -"description": "The following are multiple choice questions (with answers) about philosophy.\n\ - \n" -"group": "mmlu_alt_ov_02b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_philosophy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_prehistory.yaml deleted file mode 100644 index 4c6498ec..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_prehistory.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "prehistory" -"description": "The following are multiple choice questions (with answers) about prehistory.\n\ - \n" -"group": "mmlu_alt_ov_02b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_prehistory" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_professional_accounting.yaml deleted file mode 100644 index 1d85788b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_professional_accounting.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_accounting" -"description": "The following are multiple choice questions (with answers) about professional\ - \ accounting.\n\n" -"group": "mmlu_alt_ov_02b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_professional_accounting" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_professional_law.yaml deleted file mode 100644 index 1e73bdb4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_professional_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_law" -"description": "The following are multiple choice questions (with answers) about professional\ - \ law.\n\n" -"group": "mmlu_alt_ov_02b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_professional_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_professional_medicine.yaml deleted file mode 100644 index f3a83761..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_professional_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_medicine" -"description": "The following are multiple choice questions (with answers) about professional\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_02b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_professional_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_professional_psychology.yaml deleted file mode 100644 index a64b92e3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_professional_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_psychology" -"description": "The following are multiple choice questions (with answers) about professional\ - \ psychology.\n\n" -"group": "mmlu_alt_ov_02b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_professional_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_public_relations.yaml deleted file mode 100644 index 11c8dafe..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_public_relations.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "public_relations" -"description": "The following are multiple choice questions (with answers) about public\ - \ relations.\n\n" -"group": "mmlu_alt_ov_02b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_public_relations" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_security_studies.yaml deleted file mode 100644 index cbe15e92..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_security_studies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "security_studies" -"description": "The following are multiple choice questions (with answers) about security\ - \ studies.\n\n" -"group": "mmlu_alt_ov_02b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_security_studies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_sociology.yaml deleted file mode 100644 index d368d695..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_sociology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sociology" -"description": "The following are multiple choice questions (with answers) about sociology.\n\ - \n" -"group": "mmlu_alt_ov_02b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_sociology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_us_foreign_policy.yaml deleted file mode 100644 index 8967a97e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_us_foreign_policy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "us_foreign_policy" -"description": "The following are multiple choice questions (with answers) about us\ - \ foreign policy.\n\n" -"group": "mmlu_alt_ov_02b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_virology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_virology.yaml deleted file mode 100644 index 8e257c1e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_virology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "virology" -"description": "The following are multiple choice questions (with answers) about virology.\n\ - \n" -"group": "mmlu_alt_ov_02b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_virology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_world_religions.yaml deleted file mode 100644 index 6c2529c1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/b/mmlu_world_religions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "world_religions" -"description": "The following are multiple choice questions (with answers) about world\ - \ religions.\n\n" -"group": "mmlu_alt_ov_02b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02b_world_religions" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/_mmlu.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/_mmlu.yaml deleted file mode 100644 index b203b6af..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/_mmlu.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: mmlu_alt_ov_02c -task: - - mmlu_alt_ov_02c_stem - - mmlu_alt_ov_02c_other - - mmlu_alt_ov_02c_social_sciences - - mmlu_alt_ov_02c_humanities diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/_template_yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/_template_yaml deleted file mode 100644 index 15784a85..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/_template_yaml +++ /dev/null @@ -1,11 +0,0 @@ -dataset_path: hails/mmlu_no_train -test_split: test -fewshot_split: dev -output_type: multiple_choice -doc_to_text: !function ../../../styles.template_02 -doc_to_choice: !function ../../../styles.choice_02c -doc_to_target: answer -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_abstract_algebra.yaml deleted file mode 100644 index 2995969b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_abstract_algebra.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "abstract_algebra" -"description": "The following are multiple choice questions (with answers) about abstract\ - \ algebra.\n\n" -"group": "mmlu_alt_ov_02c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_anatomy.yaml deleted file mode 100644 index 79df6882..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_anatomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "anatomy" -"description": "The following are multiple choice questions (with answers) about anatomy.\n\ - \n" -"group": "mmlu_alt_ov_02c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_anatomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_astronomy.yaml deleted file mode 100644 index f637b8f0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_astronomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "astronomy" -"description": "The following are multiple choice questions (with answers) about astronomy.\n\ - \n" -"group": "mmlu_alt_ov_02c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_astronomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_business_ethics.yaml deleted file mode 100644 index f5719fe5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_business_ethics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "business_ethics" -"description": "The following are multiple choice questions (with answers) about business\ - \ ethics.\n\n" -"group": "mmlu_alt_ov_02c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_business_ethics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_clinical_knowledge.yaml deleted file mode 100644 index 69a546df..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_clinical_knowledge.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "clinical_knowledge" -"description": "The following are multiple choice questions (with answers) about clinical\ - \ knowledge.\n\n" -"group": "mmlu_alt_ov_02c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_college_biology.yaml deleted file mode 100644 index a9d4f001..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_college_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_biology" -"description": "The following are multiple choice questions (with answers) about college\ - \ biology.\n\n" -"group": "mmlu_alt_ov_02c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_college_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_college_chemistry.yaml deleted file mode 100644 index 66b92c48..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_college_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_chemistry" -"description": "The following are multiple choice questions (with answers) about college\ - \ chemistry.\n\n" -"group": "mmlu_alt_ov_02c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_college_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_college_computer_science.yaml deleted file mode 100644 index fab1db74..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_college_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_computer_science" -"description": "The following are multiple choice questions (with answers) about college\ - \ computer science.\n\n" -"group": "mmlu_alt_ov_02c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_college_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_college_mathematics.yaml deleted file mode 100644 index 0448b27e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_college_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_mathematics" -"description": "The following are multiple choice questions (with answers) about college\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_02c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_college_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_college_medicine.yaml deleted file mode 100644 index a760b223..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_college_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_medicine" -"description": "The following are multiple choice questions (with answers) about college\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_02c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_college_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_college_physics.yaml deleted file mode 100644 index 59008d60..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_college_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_physics" -"description": "The following are multiple choice questions (with answers) about college\ - \ physics.\n\n" -"group": "mmlu_alt_ov_02c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_college_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_computer_security.yaml deleted file mode 100644 index c0f725bb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_computer_security.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "computer_security" -"description": "The following are multiple choice questions (with answers) about computer\ - \ security.\n\n" -"group": "mmlu_alt_ov_02c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_computer_security" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_conceptual_physics.yaml deleted file mode 100644 index 3a05daea..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_conceptual_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "conceptual_physics" -"description": "The following are multiple choice questions (with answers) about conceptual\ - \ physics.\n\n" -"group": "mmlu_alt_ov_02c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_econometrics.yaml deleted file mode 100644 index 2d60b394..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_econometrics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "econometrics" -"description": "The following are multiple choice questions (with answers) about econometrics.\n\ - \n" -"group": "mmlu_alt_ov_02c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_econometrics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_electrical_engineering.yaml deleted file mode 100644 index 4616d29f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_electrical_engineering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "electrical_engineering" -"description": "The following are multiple choice questions (with answers) about electrical\ - \ engineering.\n\n" -"group": "mmlu_alt_ov_02c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_elementary_mathematics.yaml deleted file mode 100644 index f1441a0e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_elementary_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "elementary_mathematics" -"description": "The following are multiple choice questions (with answers) about elementary\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_02c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_formal_logic.yaml deleted file mode 100644 index 0593170a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_formal_logic.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_logic" -"description": "The following are multiple choice questions (with answers) about formal\ - \ logic.\n\n" -"group": "mmlu_alt_ov_02c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_formal_logic" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_global_facts.yaml deleted file mode 100644 index e59ff221..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_global_facts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "global_facts" -"description": "The following are multiple choice questions (with answers) about global\ - \ facts.\n\n" -"group": "mmlu_alt_ov_02c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_global_facts" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_biology.yaml deleted file mode 100644 index 733994e8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_biology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school biology.\n\n" -"group": "mmlu_alt_ov_02c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_high_school_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_chemistry.yaml deleted file mode 100644 index 583722f4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_chemistry" -"description": "The following are multiple choice questions (with answers) about high\ - \ school chemistry.\n\n" -"group": "mmlu_alt_ov_02c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_computer_science.yaml deleted file mode 100644 index 9480cef1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_computer_science" -"description": "The following are multiple choice questions (with answers) about high\ - \ school computer science.\n\n" -"group": "mmlu_alt_ov_02c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_european_history.yaml deleted file mode 100644 index 0e34274a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_european_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_european_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school european history.\n\n" -"group": "mmlu_alt_ov_02c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_geography.yaml deleted file mode 100644 index db513bef..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_geography.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_geography" -"description": "The following are multiple choice questions (with answers) about high\ - \ school geography.\n\n" -"group": "mmlu_alt_ov_02c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_high_school_geography" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_government_and_politics.yaml deleted file mode 100644 index ba41acc3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_government_and_politics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_government_and_politics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school government and politics.\n\n" -"group": "mmlu_alt_ov_02c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_macroeconomics.yaml deleted file mode 100644 index 155d67d1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_macroeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_macroeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school macroeconomics.\n\n" -"group": "mmlu_alt_ov_02c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_mathematics.yaml deleted file mode 100644 index 1dfd8e70..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_mathematics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school mathematics.\n\n" -"group": "mmlu_alt_ov_02c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_microeconomics.yaml deleted file mode 100644 index d5da992a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_microeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_microeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school microeconomics.\n\n" -"group": "mmlu_alt_ov_02c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_physics.yaml deleted file mode 100644 index 7f7f6e47..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_physics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school physics.\n\n" -"group": "mmlu_alt_ov_02c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_high_school_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_psychology.yaml deleted file mode 100644 index e6d7fc2f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_psychology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school psychology.\n\n" -"group": "mmlu_alt_ov_02c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_statistics.yaml deleted file mode 100644 index 58553014..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_statistics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_statistics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school statistics.\n\n" -"group": "mmlu_alt_ov_02c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_us_history.yaml deleted file mode 100644 index 74d97255..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_us_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_us_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school us history.\n\n" -"group": "mmlu_alt_ov_02c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_world_history.yaml deleted file mode 100644 index c2a85673..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_high_school_world_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_world_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school world history.\n\n" -"group": "mmlu_alt_ov_02c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_human_aging.yaml deleted file mode 100644 index e44b95c4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_human_aging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_aging" -"description": "The following are multiple choice questions (with answers) about human\ - \ aging.\n\n" -"group": "mmlu_alt_ov_02c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_human_aging" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_human_sexuality.yaml deleted file mode 100644 index dca8f5d6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_human_sexuality.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_sexuality" -"description": "The following are multiple choice questions (with answers) about human\ - \ sexuality.\n\n" -"group": "mmlu_alt_ov_02c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_human_sexuality" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_international_law.yaml deleted file mode 100644 index 9b6e4a88..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_international_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "international_law" -"description": "The following are multiple choice questions (with answers) about international\ - \ law.\n\n" -"group": "mmlu_alt_ov_02c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_international_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_jurisprudence.yaml deleted file mode 100644 index 6f785e0e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_jurisprudence.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "jurisprudence" -"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ - \n" -"group": "mmlu_alt_ov_02c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_jurisprudence" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_logical_fallacies.yaml deleted file mode 100644 index d610efbc..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_logical_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "logical_fallacies" -"description": "The following are multiple choice questions (with answers) about logical\ - \ fallacies.\n\n" -"group": "mmlu_alt_ov_02c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_machine_learning.yaml deleted file mode 100644 index 791ec4f3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_machine_learning.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "machine_learning" -"description": "The following are multiple choice questions (with answers) about machine\ - \ learning.\n\n" -"group": "mmlu_alt_ov_02c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_machine_learning" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_management.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_management.yaml deleted file mode 100644 index 5a49c529..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_management.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "management" -"description": "The following are multiple choice questions (with answers) about management.\n\ - \n" -"group": "mmlu_alt_ov_02c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_management" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_marketing.yaml deleted file mode 100644 index c735a6ba..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_marketing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "marketing" -"description": "The following are multiple choice questions (with answers) about marketing.\n\ - \n" -"group": "mmlu_alt_ov_02c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_marketing" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_medical_genetics.yaml deleted file mode 100644 index e8b331eb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_medical_genetics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "medical_genetics" -"description": "The following are multiple choice questions (with answers) about medical\ - \ genetics.\n\n" -"group": "mmlu_alt_ov_02c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_medical_genetics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_miscellaneous.yaml deleted file mode 100644 index 9de856e1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_miscellaneous.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "miscellaneous" -"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ - \n" -"group": "mmlu_alt_ov_02c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_miscellaneous" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_moral_disputes.yaml deleted file mode 100644 index d8770f2a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_moral_disputes.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_disputes" -"description": "The following are multiple choice questions (with answers) about moral\ - \ disputes.\n\n" -"group": "mmlu_alt_ov_02c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_moral_disputes" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_moral_scenarios.yaml deleted file mode 100644 index b5287479..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_moral_scenarios.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_scenarios" -"description": "The following are multiple choice questions (with answers) about moral\ - \ scenarios.\n\n" -"group": "mmlu_alt_ov_02c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_nutrition.yaml deleted file mode 100644 index db79e050..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_nutrition.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "nutrition" -"description": "The following are multiple choice questions (with answers) about nutrition.\n\ - \n" -"group": "mmlu_alt_ov_02c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_nutrition" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_philosophy.yaml deleted file mode 100644 index 939e94b5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_philosophy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "philosophy" -"description": "The following are multiple choice questions (with answers) about philosophy.\n\ - \n" -"group": "mmlu_alt_ov_02c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_philosophy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_prehistory.yaml deleted file mode 100644 index 04432dae..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_prehistory.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "prehistory" -"description": "The following are multiple choice questions (with answers) about prehistory.\n\ - \n" -"group": "mmlu_alt_ov_02c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_prehistory" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_professional_accounting.yaml deleted file mode 100644 index 5ef6434b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_professional_accounting.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_accounting" -"description": "The following are multiple choice questions (with answers) about professional\ - \ accounting.\n\n" -"group": "mmlu_alt_ov_02c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_professional_accounting" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_professional_law.yaml deleted file mode 100644 index 64f9c536..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_professional_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_law" -"description": "The following are multiple choice questions (with answers) about professional\ - \ law.\n\n" -"group": "mmlu_alt_ov_02c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_professional_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_professional_medicine.yaml deleted file mode 100644 index 741176f5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_professional_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_medicine" -"description": "The following are multiple choice questions (with answers) about professional\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_02c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_professional_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_professional_psychology.yaml deleted file mode 100644 index bc25a5de..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_professional_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_psychology" -"description": "The following are multiple choice questions (with answers) about professional\ - \ psychology.\n\n" -"group": "mmlu_alt_ov_02c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_professional_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_public_relations.yaml deleted file mode 100644 index f4148aea..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_public_relations.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "public_relations" -"description": "The following are multiple choice questions (with answers) about public\ - \ relations.\n\n" -"group": "mmlu_alt_ov_02c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_public_relations" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_security_studies.yaml deleted file mode 100644 index ff3d0a17..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_security_studies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "security_studies" -"description": "The following are multiple choice questions (with answers) about security\ - \ studies.\n\n" -"group": "mmlu_alt_ov_02c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_security_studies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_sociology.yaml deleted file mode 100644 index 3ee886b7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_sociology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sociology" -"description": "The following are multiple choice questions (with answers) about sociology.\n\ - \n" -"group": "mmlu_alt_ov_02c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_sociology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_us_foreign_policy.yaml deleted file mode 100644 index 35747cfc..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_us_foreign_policy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "us_foreign_policy" -"description": "The following are multiple choice questions (with answers) about us\ - \ foreign policy.\n\n" -"group": "mmlu_alt_ov_02c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_virology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_virology.yaml deleted file mode 100644 index cbb163b3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_virology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "virology" -"description": "The following are multiple choice questions (with answers) about virology.\n\ - \n" -"group": "mmlu_alt_ov_02c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_virology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_world_religions.yaml deleted file mode 100644 index 0f6ebcde..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_02/c/mmlu_world_religions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "world_religions" -"description": "The following are multiple choice questions (with answers) about world\ - \ religions.\n\n" -"group": "mmlu_alt_ov_02c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_02c_world_religions" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/_mmlu_ov_03.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/_mmlu_ov_03.yaml deleted file mode 100644 index 92c61b2f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/_mmlu_ov_03.yaml +++ /dev/null @@ -1,5 +0,0 @@ -group: mmlu_alt_ov_03 -task: - - mmlu_alt_ov_03a - - mmlu_alt_ov_03b - - mmlu_alt_ov_03c diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/_mmlu.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/_mmlu.yaml deleted file mode 100644 index aec0d599..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/_mmlu.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: mmlu_alt_ov_03a -task: - - mmlu_alt_ov_03a_stem - - mmlu_alt_ov_03a_other - - mmlu_alt_ov_03a_social_sciences - - mmlu_alt_ov_03a_humanities diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/_template_yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/_template_yaml deleted file mode 100644 index dff6a860..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/_template_yaml +++ /dev/null @@ -1,11 +0,0 @@ -dataset_path: hails/mmlu_no_train -test_split: test -fewshot_split: dev -output_type: multiple_choice -doc_to_text: !function ../../../styles.template_03 -doc_to_choice: !function ../../../styles.choice_03a -doc_to_target: answer -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_abstract_algebra.yaml deleted file mode 100644 index 2dc68f41..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_abstract_algebra.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "abstract_algebra" -"description": "The following are multiple choice questions (with answers) about abstract\ - \ algebra.\n\n" -"group": "mmlu_alt_ov_03a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_anatomy.yaml deleted file mode 100644 index 3135aeba..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_anatomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "anatomy" -"description": "The following are multiple choice questions (with answers) about anatomy.\n\ - \n" -"group": "mmlu_alt_ov_03a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_anatomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_astronomy.yaml deleted file mode 100644 index 35db0fd5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_astronomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "astronomy" -"description": "The following are multiple choice questions (with answers) about astronomy.\n\ - \n" -"group": "mmlu_alt_ov_03a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_astronomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_business_ethics.yaml deleted file mode 100644 index ca43852c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_business_ethics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "business_ethics" -"description": "The following are multiple choice questions (with answers) about business\ - \ ethics.\n\n" -"group": "mmlu_alt_ov_03a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_business_ethics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_clinical_knowledge.yaml deleted file mode 100644 index ee884e24..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_clinical_knowledge.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "clinical_knowledge" -"description": "The following are multiple choice questions (with answers) about clinical\ - \ knowledge.\n\n" -"group": "mmlu_alt_ov_03a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_college_biology.yaml deleted file mode 100644 index 7ae31d42..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_college_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_biology" -"description": "The following are multiple choice questions (with answers) about college\ - \ biology.\n\n" -"group": "mmlu_alt_ov_03a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_college_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_college_chemistry.yaml deleted file mode 100644 index fd30fd08..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_college_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_chemistry" -"description": "The following are multiple choice questions (with answers) about college\ - \ chemistry.\n\n" -"group": "mmlu_alt_ov_03a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_college_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_college_computer_science.yaml deleted file mode 100644 index bd74f254..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_college_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_computer_science" -"description": "The following are multiple choice questions (with answers) about college\ - \ computer science.\n\n" -"group": "mmlu_alt_ov_03a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_college_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_college_mathematics.yaml deleted file mode 100644 index e45be70d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_college_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_mathematics" -"description": "The following are multiple choice questions (with answers) about college\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_03a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_college_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_college_medicine.yaml deleted file mode 100644 index 391a6849..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_college_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_medicine" -"description": "The following are multiple choice questions (with answers) about college\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_03a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_college_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_college_physics.yaml deleted file mode 100644 index 308cbfc7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_college_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_physics" -"description": "The following are multiple choice questions (with answers) about college\ - \ physics.\n\n" -"group": "mmlu_alt_ov_03a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_college_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_computer_security.yaml deleted file mode 100644 index bd155bea..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_computer_security.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "computer_security" -"description": "The following are multiple choice questions (with answers) about computer\ - \ security.\n\n" -"group": "mmlu_alt_ov_03a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_computer_security" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_conceptual_physics.yaml deleted file mode 100644 index 0ff45f92..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_conceptual_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "conceptual_physics" -"description": "The following are multiple choice questions (with answers) about conceptual\ - \ physics.\n\n" -"group": "mmlu_alt_ov_03a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_econometrics.yaml deleted file mode 100644 index 6a1b2b26..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_econometrics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "econometrics" -"description": "The following are multiple choice questions (with answers) about econometrics.\n\ - \n" -"group": "mmlu_alt_ov_03a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_econometrics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_electrical_engineering.yaml deleted file mode 100644 index c7c53393..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_electrical_engineering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "electrical_engineering" -"description": "The following are multiple choice questions (with answers) about electrical\ - \ engineering.\n\n" -"group": "mmlu_alt_ov_03a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_elementary_mathematics.yaml deleted file mode 100644 index b3e8eb0a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_elementary_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "elementary_mathematics" -"description": "The following are multiple choice questions (with answers) about elementary\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_03a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_formal_logic.yaml deleted file mode 100644 index c7819c38..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_formal_logic.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_logic" -"description": "The following are multiple choice questions (with answers) about formal\ - \ logic.\n\n" -"group": "mmlu_alt_ov_03a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_formal_logic" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_global_facts.yaml deleted file mode 100644 index 8aadcf3d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_global_facts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "global_facts" -"description": "The following are multiple choice questions (with answers) about global\ - \ facts.\n\n" -"group": "mmlu_alt_ov_03a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_global_facts" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_biology.yaml deleted file mode 100644 index d4a6284f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_biology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school biology.\n\n" -"group": "mmlu_alt_ov_03a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_high_school_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_chemistry.yaml deleted file mode 100644 index 8709ea4f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_chemistry" -"description": "The following are multiple choice questions (with answers) about high\ - \ school chemistry.\n\n" -"group": "mmlu_alt_ov_03a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_computer_science.yaml deleted file mode 100644 index 5323a1b4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_computer_science" -"description": "The following are multiple choice questions (with answers) about high\ - \ school computer science.\n\n" -"group": "mmlu_alt_ov_03a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_european_history.yaml deleted file mode 100644 index 8dd43ab5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_european_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_european_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school european history.\n\n" -"group": "mmlu_alt_ov_03a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_geography.yaml deleted file mode 100644 index 42929928..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_geography.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_geography" -"description": "The following are multiple choice questions (with answers) about high\ - \ school geography.\n\n" -"group": "mmlu_alt_ov_03a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_high_school_geography" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_government_and_politics.yaml deleted file mode 100644 index 67d153e0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_government_and_politics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_government_and_politics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school government and politics.\n\n" -"group": "mmlu_alt_ov_03a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_macroeconomics.yaml deleted file mode 100644 index b0dc557f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_macroeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_macroeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school macroeconomics.\n\n" -"group": "mmlu_alt_ov_03a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_mathematics.yaml deleted file mode 100644 index 001a11f4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_mathematics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school mathematics.\n\n" -"group": "mmlu_alt_ov_03a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_microeconomics.yaml deleted file mode 100644 index eab63c5d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_microeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_microeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school microeconomics.\n\n" -"group": "mmlu_alt_ov_03a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_physics.yaml deleted file mode 100644 index d5422c4b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_physics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school physics.\n\n" -"group": "mmlu_alt_ov_03a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_high_school_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_psychology.yaml deleted file mode 100644 index 80df8ad2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_psychology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school psychology.\n\n" -"group": "mmlu_alt_ov_03a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_statistics.yaml deleted file mode 100644 index 4ef4b260..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_statistics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_statistics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school statistics.\n\n" -"group": "mmlu_alt_ov_03a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_us_history.yaml deleted file mode 100644 index cc86991a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_us_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_us_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school us history.\n\n" -"group": "mmlu_alt_ov_03a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_world_history.yaml deleted file mode 100644 index 2ffa6b0d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_high_school_world_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_world_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school world history.\n\n" -"group": "mmlu_alt_ov_03a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_human_aging.yaml deleted file mode 100644 index 4f0c5e19..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_human_aging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_aging" -"description": "The following are multiple choice questions (with answers) about human\ - \ aging.\n\n" -"group": "mmlu_alt_ov_03a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_human_aging" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_human_sexuality.yaml deleted file mode 100644 index a9cc8675..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_human_sexuality.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_sexuality" -"description": "The following are multiple choice questions (with answers) about human\ - \ sexuality.\n\n" -"group": "mmlu_alt_ov_03a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_human_sexuality" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_international_law.yaml deleted file mode 100644 index d2baeb17..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_international_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "international_law" -"description": "The following are multiple choice questions (with answers) about international\ - \ law.\n\n" -"group": "mmlu_alt_ov_03a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_international_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_jurisprudence.yaml deleted file mode 100644 index 17bd9195..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_jurisprudence.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "jurisprudence" -"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ - \n" -"group": "mmlu_alt_ov_03a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_jurisprudence" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_logical_fallacies.yaml deleted file mode 100644 index 8c9b4c3b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_logical_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "logical_fallacies" -"description": "The following are multiple choice questions (with answers) about logical\ - \ fallacies.\n\n" -"group": "mmlu_alt_ov_03a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_machine_learning.yaml deleted file mode 100644 index c9e52901..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_machine_learning.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "machine_learning" -"description": "The following are multiple choice questions (with answers) about machine\ - \ learning.\n\n" -"group": "mmlu_alt_ov_03a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_machine_learning" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_management.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_management.yaml deleted file mode 100644 index a1d161c8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_management.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "management" -"description": "The following are multiple choice questions (with answers) about management.\n\ - \n" -"group": "mmlu_alt_ov_03a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_management" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_marketing.yaml deleted file mode 100644 index 6477216d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_marketing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "marketing" -"description": "The following are multiple choice questions (with answers) about marketing.\n\ - \n" -"group": "mmlu_alt_ov_03a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_marketing" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_medical_genetics.yaml deleted file mode 100644 index cc1a91c1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_medical_genetics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "medical_genetics" -"description": "The following are multiple choice questions (with answers) about medical\ - \ genetics.\n\n" -"group": "mmlu_alt_ov_03a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_medical_genetics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_miscellaneous.yaml deleted file mode 100644 index ed6b320d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_miscellaneous.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "miscellaneous" -"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ - \n" -"group": "mmlu_alt_ov_03a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_miscellaneous" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_moral_disputes.yaml deleted file mode 100644 index 2aeed4fa..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_moral_disputes.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_disputes" -"description": "The following are multiple choice questions (with answers) about moral\ - \ disputes.\n\n" -"group": "mmlu_alt_ov_03a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_moral_disputes" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_moral_scenarios.yaml deleted file mode 100644 index 878b7dfe..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_moral_scenarios.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_scenarios" -"description": "The following are multiple choice questions (with answers) about moral\ - \ scenarios.\n\n" -"group": "mmlu_alt_ov_03a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_nutrition.yaml deleted file mode 100644 index e0c4a2cb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_nutrition.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "nutrition" -"description": "The following are multiple choice questions (with answers) about nutrition.\n\ - \n" -"group": "mmlu_alt_ov_03a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_nutrition" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_philosophy.yaml deleted file mode 100644 index 4a8d7020..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_philosophy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "philosophy" -"description": "The following are multiple choice questions (with answers) about philosophy.\n\ - \n" -"group": "mmlu_alt_ov_03a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_philosophy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_prehistory.yaml deleted file mode 100644 index b32bc19d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_prehistory.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "prehistory" -"description": "The following are multiple choice questions (with answers) about prehistory.\n\ - \n" -"group": "mmlu_alt_ov_03a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_prehistory" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_professional_accounting.yaml deleted file mode 100644 index 0ae97513..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_professional_accounting.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_accounting" -"description": "The following are multiple choice questions (with answers) about professional\ - \ accounting.\n\n" -"group": "mmlu_alt_ov_03a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_professional_accounting" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_professional_law.yaml deleted file mode 100644 index 744ee067..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_professional_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_law" -"description": "The following are multiple choice questions (with answers) about professional\ - \ law.\n\n" -"group": "mmlu_alt_ov_03a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_professional_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_professional_medicine.yaml deleted file mode 100644 index cc67308a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_professional_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_medicine" -"description": "The following are multiple choice questions (with answers) about professional\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_03a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_professional_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_professional_psychology.yaml deleted file mode 100644 index e0e5dad0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_professional_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_psychology" -"description": "The following are multiple choice questions (with answers) about professional\ - \ psychology.\n\n" -"group": "mmlu_alt_ov_03a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_professional_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_public_relations.yaml deleted file mode 100644 index e26d3724..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_public_relations.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "public_relations" -"description": "The following are multiple choice questions (with answers) about public\ - \ relations.\n\n" -"group": "mmlu_alt_ov_03a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_public_relations" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_security_studies.yaml deleted file mode 100644 index 6f2d39ad..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_security_studies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "security_studies" -"description": "The following are multiple choice questions (with answers) about security\ - \ studies.\n\n" -"group": "mmlu_alt_ov_03a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_security_studies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_sociology.yaml deleted file mode 100644 index 808bc194..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_sociology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sociology" -"description": "The following are multiple choice questions (with answers) about sociology.\n\ - \n" -"group": "mmlu_alt_ov_03a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_sociology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_us_foreign_policy.yaml deleted file mode 100644 index 5cd2a44d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_us_foreign_policy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "us_foreign_policy" -"description": "The following are multiple choice questions (with answers) about us\ - \ foreign policy.\n\n" -"group": "mmlu_alt_ov_03a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_virology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_virology.yaml deleted file mode 100644 index 542cbecd..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_virology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "virology" -"description": "The following are multiple choice questions (with answers) about virology.\n\ - \n" -"group": "mmlu_alt_ov_03a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_virology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_world_religions.yaml deleted file mode 100644 index 9383a90d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/a/mmlu_world_religions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "world_religions" -"description": "The following are multiple choice questions (with answers) about world\ - \ religions.\n\n" -"group": "mmlu_alt_ov_03a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03a_world_religions" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/_mmlu.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/_mmlu.yaml deleted file mode 100644 index 97b9deff..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/_mmlu.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: mmlu_alt_ov_03b -task: - - mmlu_alt_ov_03b_stem - - mmlu_alt_ov_03b_other - - mmlu_alt_ov_03b_social_sciences - - mmlu_alt_ov_03b_humanities diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/_template_yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/_template_yaml deleted file mode 100644 index 9c2f674d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/_template_yaml +++ /dev/null @@ -1,11 +0,0 @@ -dataset_path: hails/mmlu_no_train -test_split: test -fewshot_split: dev -output_type: multiple_choice -doc_to_text: !function ../../../styles.template_03 -doc_to_choice: !function ../../../styles.choice_03b -doc_to_target: answer -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_abstract_algebra.yaml deleted file mode 100644 index 8125f799..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_abstract_algebra.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "abstract_algebra" -"description": "The following are multiple choice questions (with answers) about abstract\ - \ algebra.\n\n" -"group": "mmlu_alt_ov_03b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_anatomy.yaml deleted file mode 100644 index 49e6b44a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_anatomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "anatomy" -"description": "The following are multiple choice questions (with answers) about anatomy.\n\ - \n" -"group": "mmlu_alt_ov_03b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_anatomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_astronomy.yaml deleted file mode 100644 index c08ddb38..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_astronomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "astronomy" -"description": "The following are multiple choice questions (with answers) about astronomy.\n\ - \n" -"group": "mmlu_alt_ov_03b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_astronomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_business_ethics.yaml deleted file mode 100644 index 8e0c1a93..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_business_ethics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "business_ethics" -"description": "The following are multiple choice questions (with answers) about business\ - \ ethics.\n\n" -"group": "mmlu_alt_ov_03b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_business_ethics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_clinical_knowledge.yaml deleted file mode 100644 index 8a58597a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_clinical_knowledge.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "clinical_knowledge" -"description": "The following are multiple choice questions (with answers) about clinical\ - \ knowledge.\n\n" -"group": "mmlu_alt_ov_03b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_college_biology.yaml deleted file mode 100644 index 7fd86165..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_college_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_biology" -"description": "The following are multiple choice questions (with answers) about college\ - \ biology.\n\n" -"group": "mmlu_alt_ov_03b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_college_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_college_chemistry.yaml deleted file mode 100644 index 3bae9bb7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_college_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_chemistry" -"description": "The following are multiple choice questions (with answers) about college\ - \ chemistry.\n\n" -"group": "mmlu_alt_ov_03b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_college_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_college_computer_science.yaml deleted file mode 100644 index cb6c5b6e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_college_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_computer_science" -"description": "The following are multiple choice questions (with answers) about college\ - \ computer science.\n\n" -"group": "mmlu_alt_ov_03b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_college_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_college_mathematics.yaml deleted file mode 100644 index 8a8ce24d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_college_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_mathematics" -"description": "The following are multiple choice questions (with answers) about college\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_03b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_college_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_college_medicine.yaml deleted file mode 100644 index 2586b9b4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_college_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_medicine" -"description": "The following are multiple choice questions (with answers) about college\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_03b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_college_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_college_physics.yaml deleted file mode 100644 index 8cd632d0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_college_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_physics" -"description": "The following are multiple choice questions (with answers) about college\ - \ physics.\n\n" -"group": "mmlu_alt_ov_03b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_college_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_computer_security.yaml deleted file mode 100644 index 85cf6203..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_computer_security.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "computer_security" -"description": "The following are multiple choice questions (with answers) about computer\ - \ security.\n\n" -"group": "mmlu_alt_ov_03b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_computer_security" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_conceptual_physics.yaml deleted file mode 100644 index 6d553745..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_conceptual_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "conceptual_physics" -"description": "The following are multiple choice questions (with answers) about conceptual\ - \ physics.\n\n" -"group": "mmlu_alt_ov_03b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_econometrics.yaml deleted file mode 100644 index ba8a62e5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_econometrics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "econometrics" -"description": "The following are multiple choice questions (with answers) about econometrics.\n\ - \n" -"group": "mmlu_alt_ov_03b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_econometrics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_electrical_engineering.yaml deleted file mode 100644 index 00974ad8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_electrical_engineering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "electrical_engineering" -"description": "The following are multiple choice questions (with answers) about electrical\ - \ engineering.\n\n" -"group": "mmlu_alt_ov_03b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_elementary_mathematics.yaml deleted file mode 100644 index 42bc502a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_elementary_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "elementary_mathematics" -"description": "The following are multiple choice questions (with answers) about elementary\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_03b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_formal_logic.yaml deleted file mode 100644 index 03478823..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_formal_logic.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_logic" -"description": "The following are multiple choice questions (with answers) about formal\ - \ logic.\n\n" -"group": "mmlu_alt_ov_03b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_formal_logic" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_global_facts.yaml deleted file mode 100644 index 77632fe8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_global_facts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "global_facts" -"description": "The following are multiple choice questions (with answers) about global\ - \ facts.\n\n" -"group": "mmlu_alt_ov_03b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_global_facts" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_biology.yaml deleted file mode 100644 index 72c770c2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_biology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school biology.\n\n" -"group": "mmlu_alt_ov_03b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_high_school_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_chemistry.yaml deleted file mode 100644 index 1a9dfcdf..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_chemistry" -"description": "The following are multiple choice questions (with answers) about high\ - \ school chemistry.\n\n" -"group": "mmlu_alt_ov_03b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_computer_science.yaml deleted file mode 100644 index e9fe7b8d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_computer_science" -"description": "The following are multiple choice questions (with answers) about high\ - \ school computer science.\n\n" -"group": "mmlu_alt_ov_03b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_european_history.yaml deleted file mode 100644 index 6273d5f4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_european_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_european_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school european history.\n\n" -"group": "mmlu_alt_ov_03b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_geography.yaml deleted file mode 100644 index fbcd3c24..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_geography.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_geography" -"description": "The following are multiple choice questions (with answers) about high\ - \ school geography.\n\n" -"group": "mmlu_alt_ov_03b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_high_school_geography" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_government_and_politics.yaml deleted file mode 100644 index a4cdbf20..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_government_and_politics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_government_and_politics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school government and politics.\n\n" -"group": "mmlu_alt_ov_03b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_macroeconomics.yaml deleted file mode 100644 index b57f64fc..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_macroeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_macroeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school macroeconomics.\n\n" -"group": "mmlu_alt_ov_03b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_mathematics.yaml deleted file mode 100644 index 6fe0f2fc..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_mathematics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school mathematics.\n\n" -"group": "mmlu_alt_ov_03b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_microeconomics.yaml deleted file mode 100644 index 30e29284..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_microeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_microeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school microeconomics.\n\n" -"group": "mmlu_alt_ov_03b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_physics.yaml deleted file mode 100644 index 5e587ce8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_physics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school physics.\n\n" -"group": "mmlu_alt_ov_03b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_high_school_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_psychology.yaml deleted file mode 100644 index e8da0422..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_psychology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school psychology.\n\n" -"group": "mmlu_alt_ov_03b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_statistics.yaml deleted file mode 100644 index a6286b33..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_statistics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_statistics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school statistics.\n\n" -"group": "mmlu_alt_ov_03b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_us_history.yaml deleted file mode 100644 index 2e4fa836..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_us_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_us_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school us history.\n\n" -"group": "mmlu_alt_ov_03b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_world_history.yaml deleted file mode 100644 index da65a5be..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_high_school_world_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_world_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school world history.\n\n" -"group": "mmlu_alt_ov_03b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_human_aging.yaml deleted file mode 100644 index 79822f51..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_human_aging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_aging" -"description": "The following are multiple choice questions (with answers) about human\ - \ aging.\n\n" -"group": "mmlu_alt_ov_03b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_human_aging" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_human_sexuality.yaml deleted file mode 100644 index 3c89a184..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_human_sexuality.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_sexuality" -"description": "The following are multiple choice questions (with answers) about human\ - \ sexuality.\n\n" -"group": "mmlu_alt_ov_03b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_human_sexuality" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_international_law.yaml deleted file mode 100644 index 91558fa1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_international_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "international_law" -"description": "The following are multiple choice questions (with answers) about international\ - \ law.\n\n" -"group": "mmlu_alt_ov_03b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_international_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_jurisprudence.yaml deleted file mode 100644 index 2122f7e2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_jurisprudence.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "jurisprudence" -"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ - \n" -"group": "mmlu_alt_ov_03b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_jurisprudence" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_logical_fallacies.yaml deleted file mode 100644 index 08315929..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_logical_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "logical_fallacies" -"description": "The following are multiple choice questions (with answers) about logical\ - \ fallacies.\n\n" -"group": "mmlu_alt_ov_03b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_machine_learning.yaml deleted file mode 100644 index 8ef6877c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_machine_learning.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "machine_learning" -"description": "The following are multiple choice questions (with answers) about machine\ - \ learning.\n\n" -"group": "mmlu_alt_ov_03b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_machine_learning" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_management.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_management.yaml deleted file mode 100644 index 83dc821d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_management.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "management" -"description": "The following are multiple choice questions (with answers) about management.\n\ - \n" -"group": "mmlu_alt_ov_03b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_management" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_marketing.yaml deleted file mode 100644 index fdb0b828..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_marketing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "marketing" -"description": "The following are multiple choice questions (with answers) about marketing.\n\ - \n" -"group": "mmlu_alt_ov_03b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_marketing" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_medical_genetics.yaml deleted file mode 100644 index 17de8f0f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_medical_genetics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "medical_genetics" -"description": "The following are multiple choice questions (with answers) about medical\ - \ genetics.\n\n" -"group": "mmlu_alt_ov_03b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_medical_genetics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_miscellaneous.yaml deleted file mode 100644 index 6537662e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_miscellaneous.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "miscellaneous" -"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ - \n" -"group": "mmlu_alt_ov_03b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_miscellaneous" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_moral_disputes.yaml deleted file mode 100644 index 42e23d04..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_moral_disputes.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_disputes" -"description": "The following are multiple choice questions (with answers) about moral\ - \ disputes.\n\n" -"group": "mmlu_alt_ov_03b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_moral_disputes" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_moral_scenarios.yaml deleted file mode 100644 index 5f946143..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_moral_scenarios.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_scenarios" -"description": "The following are multiple choice questions (with answers) about moral\ - \ scenarios.\n\n" -"group": "mmlu_alt_ov_03b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_nutrition.yaml deleted file mode 100644 index b786e7d8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_nutrition.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "nutrition" -"description": "The following are multiple choice questions (with answers) about nutrition.\n\ - \n" -"group": "mmlu_alt_ov_03b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_nutrition" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_philosophy.yaml deleted file mode 100644 index a8d1e4db..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_philosophy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "philosophy" -"description": "The following are multiple choice questions (with answers) about philosophy.\n\ - \n" -"group": "mmlu_alt_ov_03b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_philosophy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_prehistory.yaml deleted file mode 100644 index d2710da1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_prehistory.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "prehistory" -"description": "The following are multiple choice questions (with answers) about prehistory.\n\ - \n" -"group": "mmlu_alt_ov_03b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_prehistory" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_professional_accounting.yaml deleted file mode 100644 index 65c7e021..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_professional_accounting.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_accounting" -"description": "The following are multiple choice questions (with answers) about professional\ - \ accounting.\n\n" -"group": "mmlu_alt_ov_03b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_professional_accounting" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_professional_law.yaml deleted file mode 100644 index 9c44d779..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_professional_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_law" -"description": "The following are multiple choice questions (with answers) about professional\ - \ law.\n\n" -"group": "mmlu_alt_ov_03b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_professional_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_professional_medicine.yaml deleted file mode 100644 index 3f922754..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_professional_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_medicine" -"description": "The following are multiple choice questions (with answers) about professional\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_03b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_professional_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_professional_psychology.yaml deleted file mode 100644 index 69569a9c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_professional_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_psychology" -"description": "The following are multiple choice questions (with answers) about professional\ - \ psychology.\n\n" -"group": "mmlu_alt_ov_03b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_professional_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_public_relations.yaml deleted file mode 100644 index 150151b2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_public_relations.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "public_relations" -"description": "The following are multiple choice questions (with answers) about public\ - \ relations.\n\n" -"group": "mmlu_alt_ov_03b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_public_relations" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_security_studies.yaml deleted file mode 100644 index a73326a9..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_security_studies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "security_studies" -"description": "The following are multiple choice questions (with answers) about security\ - \ studies.\n\n" -"group": "mmlu_alt_ov_03b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_security_studies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_sociology.yaml deleted file mode 100644 index 4f736efa..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_sociology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sociology" -"description": "The following are multiple choice questions (with answers) about sociology.\n\ - \n" -"group": "mmlu_alt_ov_03b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_sociology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_us_foreign_policy.yaml deleted file mode 100644 index 277c2060..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_us_foreign_policy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "us_foreign_policy" -"description": "The following are multiple choice questions (with answers) about us\ - \ foreign policy.\n\n" -"group": "mmlu_alt_ov_03b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_virology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_virology.yaml deleted file mode 100644 index 3ef400c3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_virology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "virology" -"description": "The following are multiple choice questions (with answers) about virology.\n\ - \n" -"group": "mmlu_alt_ov_03b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_virology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_world_religions.yaml deleted file mode 100644 index 86447db7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/b/mmlu_world_religions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "world_religions" -"description": "The following are multiple choice questions (with answers) about world\ - \ religions.\n\n" -"group": "mmlu_alt_ov_03b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03b_world_religions" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/_mmlu.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/_mmlu.yaml deleted file mode 100644 index 279057b1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/_mmlu.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: mmlu_alt_ov_03c -task: - - mmlu_alt_ov_03c_stem - - mmlu_alt_ov_03c_other - - mmlu_alt_ov_03c_social_sciences - - mmlu_alt_ov_03c_humanities diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/_template_yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/_template_yaml deleted file mode 100644 index 4f8725dc..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/_template_yaml +++ /dev/null @@ -1,11 +0,0 @@ -dataset_path: hails/mmlu_no_train -test_split: test -fewshot_split: dev -output_type: multiple_choice -doc_to_text: !function ../../../styles.template_03 -doc_to_choice: !function ../../../styles.choice_03c -doc_to_target: answer -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_abstract_algebra.yaml deleted file mode 100644 index 1efca54f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_abstract_algebra.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "abstract_algebra" -"description": "The following are multiple choice questions (with answers) about abstract\ - \ algebra.\n\n" -"group": "mmlu_alt_ov_03c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_anatomy.yaml deleted file mode 100644 index de405dc9..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_anatomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "anatomy" -"description": "The following are multiple choice questions (with answers) about anatomy.\n\ - \n" -"group": "mmlu_alt_ov_03c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_anatomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_astronomy.yaml deleted file mode 100644 index 84beb052..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_astronomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "astronomy" -"description": "The following are multiple choice questions (with answers) about astronomy.\n\ - \n" -"group": "mmlu_alt_ov_03c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_astronomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_business_ethics.yaml deleted file mode 100644 index 3c899071..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_business_ethics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "business_ethics" -"description": "The following are multiple choice questions (with answers) about business\ - \ ethics.\n\n" -"group": "mmlu_alt_ov_03c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_business_ethics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_clinical_knowledge.yaml deleted file mode 100644 index c1f239af..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_clinical_knowledge.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "clinical_knowledge" -"description": "The following are multiple choice questions (with answers) about clinical\ - \ knowledge.\n\n" -"group": "mmlu_alt_ov_03c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_college_biology.yaml deleted file mode 100644 index fa70ae44..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_college_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_biology" -"description": "The following are multiple choice questions (with answers) about college\ - \ biology.\n\n" -"group": "mmlu_alt_ov_03c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_college_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_college_chemistry.yaml deleted file mode 100644 index f055b583..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_college_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_chemistry" -"description": "The following are multiple choice questions (with answers) about college\ - \ chemistry.\n\n" -"group": "mmlu_alt_ov_03c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_college_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_college_computer_science.yaml deleted file mode 100644 index 8ab1f595..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_college_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_computer_science" -"description": "The following are multiple choice questions (with answers) about college\ - \ computer science.\n\n" -"group": "mmlu_alt_ov_03c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_college_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_college_mathematics.yaml deleted file mode 100644 index 13f4ff91..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_college_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_mathematics" -"description": "The following are multiple choice questions (with answers) about college\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_03c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_college_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_college_medicine.yaml deleted file mode 100644 index 7bf4c54c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_college_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_medicine" -"description": "The following are multiple choice questions (with answers) about college\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_03c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_college_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_college_physics.yaml deleted file mode 100644 index 7142e1ba..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_college_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_physics" -"description": "The following are multiple choice questions (with answers) about college\ - \ physics.\n\n" -"group": "mmlu_alt_ov_03c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_college_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_computer_security.yaml deleted file mode 100644 index 1f5702d5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_computer_security.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "computer_security" -"description": "The following are multiple choice questions (with answers) about computer\ - \ security.\n\n" -"group": "mmlu_alt_ov_03c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_computer_security" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_conceptual_physics.yaml deleted file mode 100644 index edc6abda..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_conceptual_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "conceptual_physics" -"description": "The following are multiple choice questions (with answers) about conceptual\ - \ physics.\n\n" -"group": "mmlu_alt_ov_03c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_econometrics.yaml deleted file mode 100644 index 87646535..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_econometrics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "econometrics" -"description": "The following are multiple choice questions (with answers) about econometrics.\n\ - \n" -"group": "mmlu_alt_ov_03c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_econometrics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_electrical_engineering.yaml deleted file mode 100644 index ded23bd8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_electrical_engineering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "electrical_engineering" -"description": "The following are multiple choice questions (with answers) about electrical\ - \ engineering.\n\n" -"group": "mmlu_alt_ov_03c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_elementary_mathematics.yaml deleted file mode 100644 index 0b202d83..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_elementary_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "elementary_mathematics" -"description": "The following are multiple choice questions (with answers) about elementary\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_03c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_formal_logic.yaml deleted file mode 100644 index e45b3363..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_formal_logic.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_logic" -"description": "The following are multiple choice questions (with answers) about formal\ - \ logic.\n\n" -"group": "mmlu_alt_ov_03c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_formal_logic" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_global_facts.yaml deleted file mode 100644 index a94b03d0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_global_facts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "global_facts" -"description": "The following are multiple choice questions (with answers) about global\ - \ facts.\n\n" -"group": "mmlu_alt_ov_03c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_global_facts" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_biology.yaml deleted file mode 100644 index af162db5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_biology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school biology.\n\n" -"group": "mmlu_alt_ov_03c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_high_school_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_chemistry.yaml deleted file mode 100644 index 32fe1ff2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_chemistry" -"description": "The following are multiple choice questions (with answers) about high\ - \ school chemistry.\n\n" -"group": "mmlu_alt_ov_03c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_computer_science.yaml deleted file mode 100644 index 527ca779..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_computer_science" -"description": "The following are multiple choice questions (with answers) about high\ - \ school computer science.\n\n" -"group": "mmlu_alt_ov_03c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_european_history.yaml deleted file mode 100644 index 2def5dd7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_european_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_european_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school european history.\n\n" -"group": "mmlu_alt_ov_03c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_geography.yaml deleted file mode 100644 index 9c4a8ae1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_geography.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_geography" -"description": "The following are multiple choice questions (with answers) about high\ - \ school geography.\n\n" -"group": "mmlu_alt_ov_03c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_high_school_geography" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_government_and_politics.yaml deleted file mode 100644 index 0e7c87d8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_government_and_politics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_government_and_politics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school government and politics.\n\n" -"group": "mmlu_alt_ov_03c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_macroeconomics.yaml deleted file mode 100644 index 06f527b2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_macroeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_macroeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school macroeconomics.\n\n" -"group": "mmlu_alt_ov_03c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_mathematics.yaml deleted file mode 100644 index 598e5050..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_mathematics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school mathematics.\n\n" -"group": "mmlu_alt_ov_03c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_microeconomics.yaml deleted file mode 100644 index 889c0704..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_microeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_microeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school microeconomics.\n\n" -"group": "mmlu_alt_ov_03c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_physics.yaml deleted file mode 100644 index e00f4b9c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_physics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school physics.\n\n" -"group": "mmlu_alt_ov_03c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_high_school_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_psychology.yaml deleted file mode 100644 index c8ec2348..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_psychology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school psychology.\n\n" -"group": "mmlu_alt_ov_03c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_statistics.yaml deleted file mode 100644 index 4a31f6b9..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_statistics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_statistics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school statistics.\n\n" -"group": "mmlu_alt_ov_03c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_us_history.yaml deleted file mode 100644 index 59dfeb48..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_us_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_us_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school us history.\n\n" -"group": "mmlu_alt_ov_03c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_world_history.yaml deleted file mode 100644 index a0219ce0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_high_school_world_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_world_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school world history.\n\n" -"group": "mmlu_alt_ov_03c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_human_aging.yaml deleted file mode 100644 index 3a702d72..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_human_aging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_aging" -"description": "The following are multiple choice questions (with answers) about human\ - \ aging.\n\n" -"group": "mmlu_alt_ov_03c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_human_aging" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_human_sexuality.yaml deleted file mode 100644 index eed54461..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_human_sexuality.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_sexuality" -"description": "The following are multiple choice questions (with answers) about human\ - \ sexuality.\n\n" -"group": "mmlu_alt_ov_03c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_human_sexuality" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_international_law.yaml deleted file mode 100644 index a2dd5de4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_international_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "international_law" -"description": "The following are multiple choice questions (with answers) about international\ - \ law.\n\n" -"group": "mmlu_alt_ov_03c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_international_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_jurisprudence.yaml deleted file mode 100644 index 21880b7d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_jurisprudence.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "jurisprudence" -"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ - \n" -"group": "mmlu_alt_ov_03c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_jurisprudence" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_logical_fallacies.yaml deleted file mode 100644 index 2544c245..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_logical_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "logical_fallacies" -"description": "The following are multiple choice questions (with answers) about logical\ - \ fallacies.\n\n" -"group": "mmlu_alt_ov_03c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_machine_learning.yaml deleted file mode 100644 index a5292e41..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_machine_learning.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "machine_learning" -"description": "The following are multiple choice questions (with answers) about machine\ - \ learning.\n\n" -"group": "mmlu_alt_ov_03c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_machine_learning" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_management.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_management.yaml deleted file mode 100644 index d3e62abd..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_management.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "management" -"description": "The following are multiple choice questions (with answers) about management.\n\ - \n" -"group": "mmlu_alt_ov_03c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_management" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_marketing.yaml deleted file mode 100644 index 3958944a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_marketing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "marketing" -"description": "The following are multiple choice questions (with answers) about marketing.\n\ - \n" -"group": "mmlu_alt_ov_03c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_marketing" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_medical_genetics.yaml deleted file mode 100644 index d67df1c2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_medical_genetics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "medical_genetics" -"description": "The following are multiple choice questions (with answers) about medical\ - \ genetics.\n\n" -"group": "mmlu_alt_ov_03c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_medical_genetics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_miscellaneous.yaml deleted file mode 100644 index f084f1fa..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_miscellaneous.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "miscellaneous" -"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ - \n" -"group": "mmlu_alt_ov_03c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_miscellaneous" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_moral_disputes.yaml deleted file mode 100644 index 9b48f334..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_moral_disputes.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_disputes" -"description": "The following are multiple choice questions (with answers) about moral\ - \ disputes.\n\n" -"group": "mmlu_alt_ov_03c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_moral_disputes" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_moral_scenarios.yaml deleted file mode 100644 index ab91e4ba..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_moral_scenarios.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_scenarios" -"description": "The following are multiple choice questions (with answers) about moral\ - \ scenarios.\n\n" -"group": "mmlu_alt_ov_03c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_nutrition.yaml deleted file mode 100644 index a238240d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_nutrition.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "nutrition" -"description": "The following are multiple choice questions (with answers) about nutrition.\n\ - \n" -"group": "mmlu_alt_ov_03c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_nutrition" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_philosophy.yaml deleted file mode 100644 index ed4fdb44..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_philosophy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "philosophy" -"description": "The following are multiple choice questions (with answers) about philosophy.\n\ - \n" -"group": "mmlu_alt_ov_03c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_philosophy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_prehistory.yaml deleted file mode 100644 index 83368c8e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_prehistory.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "prehistory" -"description": "The following are multiple choice questions (with answers) about prehistory.\n\ - \n" -"group": "mmlu_alt_ov_03c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_prehistory" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_professional_accounting.yaml deleted file mode 100644 index e3187a97..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_professional_accounting.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_accounting" -"description": "The following are multiple choice questions (with answers) about professional\ - \ accounting.\n\n" -"group": "mmlu_alt_ov_03c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_professional_accounting" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_professional_law.yaml deleted file mode 100644 index 16043103..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_professional_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_law" -"description": "The following are multiple choice questions (with answers) about professional\ - \ law.\n\n" -"group": "mmlu_alt_ov_03c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_professional_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_professional_medicine.yaml deleted file mode 100644 index 7071e39c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_professional_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_medicine" -"description": "The following are multiple choice questions (with answers) about professional\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_03c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_professional_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_professional_psychology.yaml deleted file mode 100644 index 0339394f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_professional_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_psychology" -"description": "The following are multiple choice questions (with answers) about professional\ - \ psychology.\n\n" -"group": "mmlu_alt_ov_03c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_professional_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_public_relations.yaml deleted file mode 100644 index 0e1a4f45..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_public_relations.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "public_relations" -"description": "The following are multiple choice questions (with answers) about public\ - \ relations.\n\n" -"group": "mmlu_alt_ov_03c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_public_relations" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_security_studies.yaml deleted file mode 100644 index 6141dbdf..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_security_studies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "security_studies" -"description": "The following are multiple choice questions (with answers) about security\ - \ studies.\n\n" -"group": "mmlu_alt_ov_03c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_security_studies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_sociology.yaml deleted file mode 100644 index 5d748f3e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_sociology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sociology" -"description": "The following are multiple choice questions (with answers) about sociology.\n\ - \n" -"group": "mmlu_alt_ov_03c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_sociology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_us_foreign_policy.yaml deleted file mode 100644 index 782cd716..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_us_foreign_policy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "us_foreign_policy" -"description": "The following are multiple choice questions (with answers) about us\ - \ foreign policy.\n\n" -"group": "mmlu_alt_ov_03c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_virology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_virology.yaml deleted file mode 100644 index d0f91a7f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_virology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "virology" -"description": "The following are multiple choice questions (with answers) about virology.\n\ - \n" -"group": "mmlu_alt_ov_03c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_virology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_world_religions.yaml deleted file mode 100644 index 1c3446b3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_03/c/mmlu_world_religions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "world_religions" -"description": "The following are multiple choice questions (with answers) about world\ - \ religions.\n\n" -"group": "mmlu_alt_ov_03c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_03c_world_religions" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/_mmlu_ov_04.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/_mmlu_ov_04.yaml deleted file mode 100644 index 6dbe0e4e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/_mmlu_ov_04.yaml +++ /dev/null @@ -1,5 +0,0 @@ -group: mmlu_alt_ov_04 -task: - - mmlu_alt_ov_04a - - mmlu_alt_ov_04b - - mmlu_alt_ov_04c diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/_mmlu.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/_mmlu.yaml deleted file mode 100644 index 4c72d1b6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/_mmlu.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: mmlu_alt_ov_04a -task: - - mmlu_alt_ov_04a_stem - - mmlu_alt_ov_04a_other - - mmlu_alt_ov_04a_social_sciences - - mmlu_alt_ov_04a_humanities diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/_template_yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/_template_yaml deleted file mode 100644 index 609cc706..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/_template_yaml +++ /dev/null @@ -1,11 +0,0 @@ -dataset_path: hails/mmlu_no_train -test_split: test -fewshot_split: dev -output_type: multiple_choice -doc_to_text: !function ../../../styles.template_04 -doc_to_choice: !function ../../../styles.choice_04a -doc_to_target: answer -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_abstract_algebra.yaml deleted file mode 100644 index 2c89e19a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_abstract_algebra.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "abstract_algebra" -"description": "The following are multiple choice questions (with answers) about abstract\ - \ algebra.\n\n" -"group": "mmlu_alt_ov_04a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_anatomy.yaml deleted file mode 100644 index 4a50e383..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_anatomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "anatomy" -"description": "The following are multiple choice questions (with answers) about anatomy.\n\ - \n" -"group": "mmlu_alt_ov_04a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_anatomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_astronomy.yaml deleted file mode 100644 index aec8048b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_astronomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "astronomy" -"description": "The following are multiple choice questions (with answers) about astronomy.\n\ - \n" -"group": "mmlu_alt_ov_04a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_astronomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_business_ethics.yaml deleted file mode 100644 index 2041d641..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_business_ethics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "business_ethics" -"description": "The following are multiple choice questions (with answers) about business\ - \ ethics.\n\n" -"group": "mmlu_alt_ov_04a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_business_ethics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_clinical_knowledge.yaml deleted file mode 100644 index 24564acb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_clinical_knowledge.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "clinical_knowledge" -"description": "The following are multiple choice questions (with answers) about clinical\ - \ knowledge.\n\n" -"group": "mmlu_alt_ov_04a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_college_biology.yaml deleted file mode 100644 index 1cd9273a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_college_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_biology" -"description": "The following are multiple choice questions (with answers) about college\ - \ biology.\n\n" -"group": "mmlu_alt_ov_04a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_college_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_college_chemistry.yaml deleted file mode 100644 index 9fd97dfe..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_college_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_chemistry" -"description": "The following are multiple choice questions (with answers) about college\ - \ chemistry.\n\n" -"group": "mmlu_alt_ov_04a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_college_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_college_computer_science.yaml deleted file mode 100644 index ede1b3b5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_college_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_computer_science" -"description": "The following are multiple choice questions (with answers) about college\ - \ computer science.\n\n" -"group": "mmlu_alt_ov_04a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_college_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_college_mathematics.yaml deleted file mode 100644 index eed15e0d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_college_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_mathematics" -"description": "The following are multiple choice questions (with answers) about college\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_04a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_college_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_college_medicine.yaml deleted file mode 100644 index 15ef7355..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_college_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_medicine" -"description": "The following are multiple choice questions (with answers) about college\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_04a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_college_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_college_physics.yaml deleted file mode 100644 index 9d04b497..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_college_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_physics" -"description": "The following are multiple choice questions (with answers) about college\ - \ physics.\n\n" -"group": "mmlu_alt_ov_04a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_college_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_computer_security.yaml deleted file mode 100644 index c7699e90..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_computer_security.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "computer_security" -"description": "The following are multiple choice questions (with answers) about computer\ - \ security.\n\n" -"group": "mmlu_alt_ov_04a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_computer_security" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_conceptual_physics.yaml deleted file mode 100644 index b44faf2a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_conceptual_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "conceptual_physics" -"description": "The following are multiple choice questions (with answers) about conceptual\ - \ physics.\n\n" -"group": "mmlu_alt_ov_04a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_econometrics.yaml deleted file mode 100644 index 9cf241d7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_econometrics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "econometrics" -"description": "The following are multiple choice questions (with answers) about econometrics.\n\ - \n" -"group": "mmlu_alt_ov_04a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_econometrics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_electrical_engineering.yaml deleted file mode 100644 index 698f8bfe..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_electrical_engineering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "electrical_engineering" -"description": "The following are multiple choice questions (with answers) about electrical\ - \ engineering.\n\n" -"group": "mmlu_alt_ov_04a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_elementary_mathematics.yaml deleted file mode 100644 index be3811c0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_elementary_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "elementary_mathematics" -"description": "The following are multiple choice questions (with answers) about elementary\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_04a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_formal_logic.yaml deleted file mode 100644 index e621f77c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_formal_logic.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_logic" -"description": "The following are multiple choice questions (with answers) about formal\ - \ logic.\n\n" -"group": "mmlu_alt_ov_04a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_formal_logic" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_global_facts.yaml deleted file mode 100644 index 1c6816b6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_global_facts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "global_facts" -"description": "The following are multiple choice questions (with answers) about global\ - \ facts.\n\n" -"group": "mmlu_alt_ov_04a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_global_facts" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_biology.yaml deleted file mode 100644 index 2951f396..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_biology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school biology.\n\n" -"group": "mmlu_alt_ov_04a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_high_school_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_chemistry.yaml deleted file mode 100644 index d2bbee0a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_chemistry" -"description": "The following are multiple choice questions (with answers) about high\ - \ school chemistry.\n\n" -"group": "mmlu_alt_ov_04a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_computer_science.yaml deleted file mode 100644 index 2404903f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_computer_science" -"description": "The following are multiple choice questions (with answers) about high\ - \ school computer science.\n\n" -"group": "mmlu_alt_ov_04a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_european_history.yaml deleted file mode 100644 index 81885bb7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_european_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_european_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school european history.\n\n" -"group": "mmlu_alt_ov_04a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_geography.yaml deleted file mode 100644 index 4362b2a5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_geography.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_geography" -"description": "The following are multiple choice questions (with answers) about high\ - \ school geography.\n\n" -"group": "mmlu_alt_ov_04a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_high_school_geography" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_government_and_politics.yaml deleted file mode 100644 index caaf9bd4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_government_and_politics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_government_and_politics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school government and politics.\n\n" -"group": "mmlu_alt_ov_04a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_macroeconomics.yaml deleted file mode 100644 index 51de1287..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_macroeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_macroeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school macroeconomics.\n\n" -"group": "mmlu_alt_ov_04a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_mathematics.yaml deleted file mode 100644 index 6355e89c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_mathematics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school mathematics.\n\n" -"group": "mmlu_alt_ov_04a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_microeconomics.yaml deleted file mode 100644 index 1fb676d2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_microeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_microeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school microeconomics.\n\n" -"group": "mmlu_alt_ov_04a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_physics.yaml deleted file mode 100644 index b2e3cf52..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_physics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school physics.\n\n" -"group": "mmlu_alt_ov_04a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_high_school_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_psychology.yaml deleted file mode 100644 index ba37f2fb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_psychology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school psychology.\n\n" -"group": "mmlu_alt_ov_04a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_statistics.yaml deleted file mode 100644 index 82fa0d32..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_statistics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_statistics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school statistics.\n\n" -"group": "mmlu_alt_ov_04a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_us_history.yaml deleted file mode 100644 index a99cda3a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_us_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_us_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school us history.\n\n" -"group": "mmlu_alt_ov_04a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_world_history.yaml deleted file mode 100644 index cd6972cb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_high_school_world_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_world_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school world history.\n\n" -"group": "mmlu_alt_ov_04a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_human_aging.yaml deleted file mode 100644 index 4525a486..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_human_aging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_aging" -"description": "The following are multiple choice questions (with answers) about human\ - \ aging.\n\n" -"group": "mmlu_alt_ov_04a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_human_aging" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_human_sexuality.yaml deleted file mode 100644 index 9bc378dc..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_human_sexuality.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_sexuality" -"description": "The following are multiple choice questions (with answers) about human\ - \ sexuality.\n\n" -"group": "mmlu_alt_ov_04a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_human_sexuality" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_international_law.yaml deleted file mode 100644 index b71af43a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_international_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "international_law" -"description": "The following are multiple choice questions (with answers) about international\ - \ law.\n\n" -"group": "mmlu_alt_ov_04a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_international_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_jurisprudence.yaml deleted file mode 100644 index d3e2eafd..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_jurisprudence.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "jurisprudence" -"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ - \n" -"group": "mmlu_alt_ov_04a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_jurisprudence" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_logical_fallacies.yaml deleted file mode 100644 index 8f662085..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_logical_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "logical_fallacies" -"description": "The following are multiple choice questions (with answers) about logical\ - \ fallacies.\n\n" -"group": "mmlu_alt_ov_04a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_machine_learning.yaml deleted file mode 100644 index 7960b3fd..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_machine_learning.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "machine_learning" -"description": "The following are multiple choice questions (with answers) about machine\ - \ learning.\n\n" -"group": "mmlu_alt_ov_04a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_machine_learning" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_management.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_management.yaml deleted file mode 100644 index 4a3eebad..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_management.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "management" -"description": "The following are multiple choice questions (with answers) about management.\n\ - \n" -"group": "mmlu_alt_ov_04a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_management" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_marketing.yaml deleted file mode 100644 index ff474e37..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_marketing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "marketing" -"description": "The following are multiple choice questions (with answers) about marketing.\n\ - \n" -"group": "mmlu_alt_ov_04a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_marketing" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_medical_genetics.yaml deleted file mode 100644 index d56452d6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_medical_genetics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "medical_genetics" -"description": "The following are multiple choice questions (with answers) about medical\ - \ genetics.\n\n" -"group": "mmlu_alt_ov_04a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_medical_genetics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_miscellaneous.yaml deleted file mode 100644 index 756bf186..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_miscellaneous.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "miscellaneous" -"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ - \n" -"group": "mmlu_alt_ov_04a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_miscellaneous" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_moral_disputes.yaml deleted file mode 100644 index 4a1d9f64..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_moral_disputes.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_disputes" -"description": "The following are multiple choice questions (with answers) about moral\ - \ disputes.\n\n" -"group": "mmlu_alt_ov_04a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_moral_disputes" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_moral_scenarios.yaml deleted file mode 100644 index 7278f5f5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_moral_scenarios.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_scenarios" -"description": "The following are multiple choice questions (with answers) about moral\ - \ scenarios.\n\n" -"group": "mmlu_alt_ov_04a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_nutrition.yaml deleted file mode 100644 index 9a87a851..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_nutrition.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "nutrition" -"description": "The following are multiple choice questions (with answers) about nutrition.\n\ - \n" -"group": "mmlu_alt_ov_04a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_nutrition" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_philosophy.yaml deleted file mode 100644 index 5a25cd46..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_philosophy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "philosophy" -"description": "The following are multiple choice questions (with answers) about philosophy.\n\ - \n" -"group": "mmlu_alt_ov_04a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_philosophy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_prehistory.yaml deleted file mode 100644 index 4d84bea4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_prehistory.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "prehistory" -"description": "The following are multiple choice questions (with answers) about prehistory.\n\ - \n" -"group": "mmlu_alt_ov_04a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_prehistory" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_professional_accounting.yaml deleted file mode 100644 index 03ae20de..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_professional_accounting.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_accounting" -"description": "The following are multiple choice questions (with answers) about professional\ - \ accounting.\n\n" -"group": "mmlu_alt_ov_04a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_professional_accounting" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_professional_law.yaml deleted file mode 100644 index 0eeddfe5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_professional_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_law" -"description": "The following are multiple choice questions (with answers) about professional\ - \ law.\n\n" -"group": "mmlu_alt_ov_04a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_professional_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_professional_medicine.yaml deleted file mode 100644 index e6f8ee82..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_professional_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_medicine" -"description": "The following are multiple choice questions (with answers) about professional\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_04a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_professional_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_professional_psychology.yaml deleted file mode 100644 index a5f3538f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_professional_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_psychology" -"description": "The following are multiple choice questions (with answers) about professional\ - \ psychology.\n\n" -"group": "mmlu_alt_ov_04a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_professional_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_public_relations.yaml deleted file mode 100644 index ca501151..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_public_relations.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "public_relations" -"description": "The following are multiple choice questions (with answers) about public\ - \ relations.\n\n" -"group": "mmlu_alt_ov_04a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_public_relations" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_security_studies.yaml deleted file mode 100644 index c4a9af05..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_security_studies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "security_studies" -"description": "The following are multiple choice questions (with answers) about security\ - \ studies.\n\n" -"group": "mmlu_alt_ov_04a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_security_studies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_sociology.yaml deleted file mode 100644 index ab0388bf..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_sociology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sociology" -"description": "The following are multiple choice questions (with answers) about sociology.\n\ - \n" -"group": "mmlu_alt_ov_04a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_sociology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_us_foreign_policy.yaml deleted file mode 100644 index 6279fec2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_us_foreign_policy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "us_foreign_policy" -"description": "The following are multiple choice questions (with answers) about us\ - \ foreign policy.\n\n" -"group": "mmlu_alt_ov_04a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_virology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_virology.yaml deleted file mode 100644 index aad84103..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_virology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "virology" -"description": "The following are multiple choice questions (with answers) about virology.\n\ - \n" -"group": "mmlu_alt_ov_04a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_virology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_world_religions.yaml deleted file mode 100644 index 8ab8f6b6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/a/mmlu_world_religions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "world_religions" -"description": "The following are multiple choice questions (with answers) about world\ - \ religions.\n\n" -"group": "mmlu_alt_ov_04a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04a_world_religions" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/_mmlu.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/_mmlu.yaml deleted file mode 100644 index 1f9887c1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/_mmlu.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: mmlu_alt_ov_04b -task: - - mmlu_alt_ov_04b_stem - - mmlu_alt_ov_04b_other - - mmlu_alt_ov_04b_social_sciences - - mmlu_alt_ov_04b_humanities diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/_template_yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/_template_yaml deleted file mode 100644 index 5d1810d1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/_template_yaml +++ /dev/null @@ -1,11 +0,0 @@ -dataset_path: hails/mmlu_no_train -test_split: test -fewshot_split: dev -output_type: multiple_choice -doc_to_text: !function ../../../styles.template_04 -doc_to_choice: !function ../../../styles.choice_04b -doc_to_target: answer -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_abstract_algebra.yaml deleted file mode 100644 index 42d301b8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_abstract_algebra.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "abstract_algebra" -"description": "The following are multiple choice questions (with answers) about abstract\ - \ algebra.\n\n" -"group": "mmlu_alt_ov_04b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_anatomy.yaml deleted file mode 100644 index 4c25ee11..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_anatomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "anatomy" -"description": "The following are multiple choice questions (with answers) about anatomy.\n\ - \n" -"group": "mmlu_alt_ov_04b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_anatomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_astronomy.yaml deleted file mode 100644 index 577d16bb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_astronomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "astronomy" -"description": "The following are multiple choice questions (with answers) about astronomy.\n\ - \n" -"group": "mmlu_alt_ov_04b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_astronomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_business_ethics.yaml deleted file mode 100644 index cf58d929..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_business_ethics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "business_ethics" -"description": "The following are multiple choice questions (with answers) about business\ - \ ethics.\n\n" -"group": "mmlu_alt_ov_04b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_business_ethics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_clinical_knowledge.yaml deleted file mode 100644 index eb7eb963..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_clinical_knowledge.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "clinical_knowledge" -"description": "The following are multiple choice questions (with answers) about clinical\ - \ knowledge.\n\n" -"group": "mmlu_alt_ov_04b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_college_biology.yaml deleted file mode 100644 index bd42c5b6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_college_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_biology" -"description": "The following are multiple choice questions (with answers) about college\ - \ biology.\n\n" -"group": "mmlu_alt_ov_04b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_college_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_college_chemistry.yaml deleted file mode 100644 index 10a9f5e4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_college_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_chemistry" -"description": "The following are multiple choice questions (with answers) about college\ - \ chemistry.\n\n" -"group": "mmlu_alt_ov_04b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_college_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_college_computer_science.yaml deleted file mode 100644 index a38fbd44..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_college_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_computer_science" -"description": "The following are multiple choice questions (with answers) about college\ - \ computer science.\n\n" -"group": "mmlu_alt_ov_04b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_college_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_college_mathematics.yaml deleted file mode 100644 index 44a15df0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_college_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_mathematics" -"description": "The following are multiple choice questions (with answers) about college\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_04b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_college_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_college_medicine.yaml deleted file mode 100644 index e5e807ac..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_college_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_medicine" -"description": "The following are multiple choice questions (with answers) about college\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_04b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_college_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_college_physics.yaml deleted file mode 100644 index da7057bc..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_college_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_physics" -"description": "The following are multiple choice questions (with answers) about college\ - \ physics.\n\n" -"group": "mmlu_alt_ov_04b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_college_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_computer_security.yaml deleted file mode 100644 index 54ad5d1f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_computer_security.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "computer_security" -"description": "The following are multiple choice questions (with answers) about computer\ - \ security.\n\n" -"group": "mmlu_alt_ov_04b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_computer_security" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_conceptual_physics.yaml deleted file mode 100644 index 72c6347d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_conceptual_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "conceptual_physics" -"description": "The following are multiple choice questions (with answers) about conceptual\ - \ physics.\n\n" -"group": "mmlu_alt_ov_04b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_econometrics.yaml deleted file mode 100644 index 607811c0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_econometrics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "econometrics" -"description": "The following are multiple choice questions (with answers) about econometrics.\n\ - \n" -"group": "mmlu_alt_ov_04b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_econometrics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_electrical_engineering.yaml deleted file mode 100644 index e7098c13..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_electrical_engineering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "electrical_engineering" -"description": "The following are multiple choice questions (with answers) about electrical\ - \ engineering.\n\n" -"group": "mmlu_alt_ov_04b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_elementary_mathematics.yaml deleted file mode 100644 index 8e9733b9..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_elementary_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "elementary_mathematics" -"description": "The following are multiple choice questions (with answers) about elementary\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_04b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_formal_logic.yaml deleted file mode 100644 index c255e82f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_formal_logic.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_logic" -"description": "The following are multiple choice questions (with answers) about formal\ - \ logic.\n\n" -"group": "mmlu_alt_ov_04b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_formal_logic" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_global_facts.yaml deleted file mode 100644 index 8d852646..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_global_facts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "global_facts" -"description": "The following are multiple choice questions (with answers) about global\ - \ facts.\n\n" -"group": "mmlu_alt_ov_04b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_global_facts" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_biology.yaml deleted file mode 100644 index 0d60b324..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_biology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school biology.\n\n" -"group": "mmlu_alt_ov_04b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_high_school_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_chemistry.yaml deleted file mode 100644 index 268f3fda..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_chemistry" -"description": "The following are multiple choice questions (with answers) about high\ - \ school chemistry.\n\n" -"group": "mmlu_alt_ov_04b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_computer_science.yaml deleted file mode 100644 index 8b0b6795..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_computer_science" -"description": "The following are multiple choice questions (with answers) about high\ - \ school computer science.\n\n" -"group": "mmlu_alt_ov_04b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_european_history.yaml deleted file mode 100644 index 5e336597..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_european_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_european_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school european history.\n\n" -"group": "mmlu_alt_ov_04b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_geography.yaml deleted file mode 100644 index 7639b9a3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_geography.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_geography" -"description": "The following are multiple choice questions (with answers) about high\ - \ school geography.\n\n" -"group": "mmlu_alt_ov_04b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_high_school_geography" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_government_and_politics.yaml deleted file mode 100644 index 97948e98..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_government_and_politics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_government_and_politics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school government and politics.\n\n" -"group": "mmlu_alt_ov_04b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_macroeconomics.yaml deleted file mode 100644 index 7cfa35b1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_macroeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_macroeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school macroeconomics.\n\n" -"group": "mmlu_alt_ov_04b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_mathematics.yaml deleted file mode 100644 index a90b6520..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_mathematics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school mathematics.\n\n" -"group": "mmlu_alt_ov_04b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_microeconomics.yaml deleted file mode 100644 index f772e76f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_microeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_microeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school microeconomics.\n\n" -"group": "mmlu_alt_ov_04b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_physics.yaml deleted file mode 100644 index ca32bdc8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_physics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school physics.\n\n" -"group": "mmlu_alt_ov_04b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_high_school_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_psychology.yaml deleted file mode 100644 index 43db99f1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_psychology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school psychology.\n\n" -"group": "mmlu_alt_ov_04b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_statistics.yaml deleted file mode 100644 index 509e0cba..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_statistics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_statistics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school statistics.\n\n" -"group": "mmlu_alt_ov_04b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_us_history.yaml deleted file mode 100644 index 680a0241..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_us_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_us_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school us history.\n\n" -"group": "mmlu_alt_ov_04b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_world_history.yaml deleted file mode 100644 index 424bffdb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_high_school_world_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_world_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school world history.\n\n" -"group": "mmlu_alt_ov_04b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_human_aging.yaml deleted file mode 100644 index 847c233e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_human_aging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_aging" -"description": "The following are multiple choice questions (with answers) about human\ - \ aging.\n\n" -"group": "mmlu_alt_ov_04b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_human_aging" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_human_sexuality.yaml deleted file mode 100644 index ab542ef1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_human_sexuality.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_sexuality" -"description": "The following are multiple choice questions (with answers) about human\ - \ sexuality.\n\n" -"group": "mmlu_alt_ov_04b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_human_sexuality" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_international_law.yaml deleted file mode 100644 index 79e2a1e3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_international_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "international_law" -"description": "The following are multiple choice questions (with answers) about international\ - \ law.\n\n" -"group": "mmlu_alt_ov_04b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_international_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_jurisprudence.yaml deleted file mode 100644 index edef94a6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_jurisprudence.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "jurisprudence" -"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ - \n" -"group": "mmlu_alt_ov_04b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_jurisprudence" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_logical_fallacies.yaml deleted file mode 100644 index 25d74687..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_logical_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "logical_fallacies" -"description": "The following are multiple choice questions (with answers) about logical\ - \ fallacies.\n\n" -"group": "mmlu_alt_ov_04b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_machine_learning.yaml deleted file mode 100644 index f139710b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_machine_learning.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "machine_learning" -"description": "The following are multiple choice questions (with answers) about machine\ - \ learning.\n\n" -"group": "mmlu_alt_ov_04b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_machine_learning" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_management.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_management.yaml deleted file mode 100644 index 94bb6aa3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_management.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "management" -"description": "The following are multiple choice questions (with answers) about management.\n\ - \n" -"group": "mmlu_alt_ov_04b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_management" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_marketing.yaml deleted file mode 100644 index 8fa3bd4b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_marketing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "marketing" -"description": "The following are multiple choice questions (with answers) about marketing.\n\ - \n" -"group": "mmlu_alt_ov_04b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_marketing" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_medical_genetics.yaml deleted file mode 100644 index 3c74cdd6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_medical_genetics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "medical_genetics" -"description": "The following are multiple choice questions (with answers) about medical\ - \ genetics.\n\n" -"group": "mmlu_alt_ov_04b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_medical_genetics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_miscellaneous.yaml deleted file mode 100644 index c3cd247d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_miscellaneous.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "miscellaneous" -"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ - \n" -"group": "mmlu_alt_ov_04b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_miscellaneous" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_moral_disputes.yaml deleted file mode 100644 index 3637540c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_moral_disputes.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_disputes" -"description": "The following are multiple choice questions (with answers) about moral\ - \ disputes.\n\n" -"group": "mmlu_alt_ov_04b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_moral_disputes" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_moral_scenarios.yaml deleted file mode 100644 index b97a78e0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_moral_scenarios.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_scenarios" -"description": "The following are multiple choice questions (with answers) about moral\ - \ scenarios.\n\n" -"group": "mmlu_alt_ov_04b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_nutrition.yaml deleted file mode 100644 index 59584190..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_nutrition.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "nutrition" -"description": "The following are multiple choice questions (with answers) about nutrition.\n\ - \n" -"group": "mmlu_alt_ov_04b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_nutrition" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_philosophy.yaml deleted file mode 100644 index 4080df18..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_philosophy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "philosophy" -"description": "The following are multiple choice questions (with answers) about philosophy.\n\ - \n" -"group": "mmlu_alt_ov_04b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_philosophy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_prehistory.yaml deleted file mode 100644 index 0101a60b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_prehistory.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "prehistory" -"description": "The following are multiple choice questions (with answers) about prehistory.\n\ - \n" -"group": "mmlu_alt_ov_04b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_prehistory" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_professional_accounting.yaml deleted file mode 100644 index a96d871a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_professional_accounting.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_accounting" -"description": "The following are multiple choice questions (with answers) about professional\ - \ accounting.\n\n" -"group": "mmlu_alt_ov_04b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_professional_accounting" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_professional_law.yaml deleted file mode 100644 index 833a4015..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_professional_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_law" -"description": "The following are multiple choice questions (with answers) about professional\ - \ law.\n\n" -"group": "mmlu_alt_ov_04b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_professional_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_professional_medicine.yaml deleted file mode 100644 index 304a57bf..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_professional_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_medicine" -"description": "The following are multiple choice questions (with answers) about professional\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_04b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_professional_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_professional_psychology.yaml deleted file mode 100644 index 84f4c03a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_professional_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_psychology" -"description": "The following are multiple choice questions (with answers) about professional\ - \ psychology.\n\n" -"group": "mmlu_alt_ov_04b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_professional_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_public_relations.yaml deleted file mode 100644 index d098cd9d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_public_relations.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "public_relations" -"description": "The following are multiple choice questions (with answers) about public\ - \ relations.\n\n" -"group": "mmlu_alt_ov_04b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_public_relations" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_security_studies.yaml deleted file mode 100644 index 6b8a5f1c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_security_studies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "security_studies" -"description": "The following are multiple choice questions (with answers) about security\ - \ studies.\n\n" -"group": "mmlu_alt_ov_04b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_security_studies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_sociology.yaml deleted file mode 100644 index e2eedc10..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_sociology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sociology" -"description": "The following are multiple choice questions (with answers) about sociology.\n\ - \n" -"group": "mmlu_alt_ov_04b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_sociology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_us_foreign_policy.yaml deleted file mode 100644 index 79a219f4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_us_foreign_policy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "us_foreign_policy" -"description": "The following are multiple choice questions (with answers) about us\ - \ foreign policy.\n\n" -"group": "mmlu_alt_ov_04b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_virology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_virology.yaml deleted file mode 100644 index 4546ce88..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_virology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "virology" -"description": "The following are multiple choice questions (with answers) about virology.\n\ - \n" -"group": "mmlu_alt_ov_04b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_virology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_world_religions.yaml deleted file mode 100644 index f1be68d4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/b/mmlu_world_religions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "world_religions" -"description": "The following are multiple choice questions (with answers) about world\ - \ religions.\n\n" -"group": "mmlu_alt_ov_04b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04b_world_religions" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/_mmlu.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/_mmlu.yaml deleted file mode 100644 index 81f8329d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/_mmlu.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: mmlu_alt_ov_04c -task: - - mmlu_alt_ov_04c_stem - - mmlu_alt_ov_04c_other - - mmlu_alt_ov_04c_social_sciences - - mmlu_alt_ov_04c_humanities diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/_template_yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/_template_yaml deleted file mode 100644 index 88af060b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/_template_yaml +++ /dev/null @@ -1,11 +0,0 @@ -dataset_path: hails/mmlu_no_train -test_split: test -fewshot_split: dev -output_type: multiple_choice -doc_to_text: !function ../../../styles.template_04 -doc_to_choice: !function ../../../styles.choice_04c -doc_to_target: answer -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_abstract_algebra.yaml deleted file mode 100644 index d5d39121..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_abstract_algebra.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "abstract_algebra" -"description": "The following are multiple choice questions (with answers) about abstract\ - \ algebra.\n\n" -"group": "mmlu_alt_ov_04c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_anatomy.yaml deleted file mode 100644 index a9b85b6f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_anatomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "anatomy" -"description": "The following are multiple choice questions (with answers) about anatomy.\n\ - \n" -"group": "mmlu_alt_ov_04c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_anatomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_astronomy.yaml deleted file mode 100644 index e10dc1d7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_astronomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "astronomy" -"description": "The following are multiple choice questions (with answers) about astronomy.\n\ - \n" -"group": "mmlu_alt_ov_04c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_astronomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_business_ethics.yaml deleted file mode 100644 index a989ebc7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_business_ethics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "business_ethics" -"description": "The following are multiple choice questions (with answers) about business\ - \ ethics.\n\n" -"group": "mmlu_alt_ov_04c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_business_ethics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_clinical_knowledge.yaml deleted file mode 100644 index 55437c1b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_clinical_knowledge.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "clinical_knowledge" -"description": "The following are multiple choice questions (with answers) about clinical\ - \ knowledge.\n\n" -"group": "mmlu_alt_ov_04c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_college_biology.yaml deleted file mode 100644 index 75d9dc12..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_college_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_biology" -"description": "The following are multiple choice questions (with answers) about college\ - \ biology.\n\n" -"group": "mmlu_alt_ov_04c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_college_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_college_chemistry.yaml deleted file mode 100644 index cb439036..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_college_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_chemistry" -"description": "The following are multiple choice questions (with answers) about college\ - \ chemistry.\n\n" -"group": "mmlu_alt_ov_04c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_college_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_college_computer_science.yaml deleted file mode 100644 index 2f5e1816..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_college_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_computer_science" -"description": "The following are multiple choice questions (with answers) about college\ - \ computer science.\n\n" -"group": "mmlu_alt_ov_04c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_college_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_college_mathematics.yaml deleted file mode 100644 index 85390337..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_college_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_mathematics" -"description": "The following are multiple choice questions (with answers) about college\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_04c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_college_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_college_medicine.yaml deleted file mode 100644 index fe8ddd58..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_college_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_medicine" -"description": "The following are multiple choice questions (with answers) about college\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_04c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_college_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_college_physics.yaml deleted file mode 100644 index 36f582d6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_college_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_physics" -"description": "The following are multiple choice questions (with answers) about college\ - \ physics.\n\n" -"group": "mmlu_alt_ov_04c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_college_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_computer_security.yaml deleted file mode 100644 index cf32b53e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_computer_security.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "computer_security" -"description": "The following are multiple choice questions (with answers) about computer\ - \ security.\n\n" -"group": "mmlu_alt_ov_04c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_computer_security" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_conceptual_physics.yaml deleted file mode 100644 index 546c74b9..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_conceptual_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "conceptual_physics" -"description": "The following are multiple choice questions (with answers) about conceptual\ - \ physics.\n\n" -"group": "mmlu_alt_ov_04c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_econometrics.yaml deleted file mode 100644 index 9ddcf20e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_econometrics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "econometrics" -"description": "The following are multiple choice questions (with answers) about econometrics.\n\ - \n" -"group": "mmlu_alt_ov_04c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_econometrics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_electrical_engineering.yaml deleted file mode 100644 index d5431c8f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_electrical_engineering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "electrical_engineering" -"description": "The following are multiple choice questions (with answers) about electrical\ - \ engineering.\n\n" -"group": "mmlu_alt_ov_04c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_elementary_mathematics.yaml deleted file mode 100644 index daeb3efc..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_elementary_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "elementary_mathematics" -"description": "The following are multiple choice questions (with answers) about elementary\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_04c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_formal_logic.yaml deleted file mode 100644 index 39912a7b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_formal_logic.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_logic" -"description": "The following are multiple choice questions (with answers) about formal\ - \ logic.\n\n" -"group": "mmlu_alt_ov_04c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_formal_logic" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_global_facts.yaml deleted file mode 100644 index c9c866d7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_global_facts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "global_facts" -"description": "The following are multiple choice questions (with answers) about global\ - \ facts.\n\n" -"group": "mmlu_alt_ov_04c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_global_facts" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_biology.yaml deleted file mode 100644 index a04054c3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_biology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school biology.\n\n" -"group": "mmlu_alt_ov_04c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_high_school_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_chemistry.yaml deleted file mode 100644 index d7b6878f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_chemistry" -"description": "The following are multiple choice questions (with answers) about high\ - \ school chemistry.\n\n" -"group": "mmlu_alt_ov_04c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_computer_science.yaml deleted file mode 100644 index 62d0a144..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_computer_science" -"description": "The following are multiple choice questions (with answers) about high\ - \ school computer science.\n\n" -"group": "mmlu_alt_ov_04c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_european_history.yaml deleted file mode 100644 index 9a950e97..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_european_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_european_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school european history.\n\n" -"group": "mmlu_alt_ov_04c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_geography.yaml deleted file mode 100644 index 2f1541bf..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_geography.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_geography" -"description": "The following are multiple choice questions (with answers) about high\ - \ school geography.\n\n" -"group": "mmlu_alt_ov_04c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_high_school_geography" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_government_and_politics.yaml deleted file mode 100644 index 6cb89ef3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_government_and_politics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_government_and_politics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school government and politics.\n\n" -"group": "mmlu_alt_ov_04c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_macroeconomics.yaml deleted file mode 100644 index f8973675..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_macroeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_macroeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school macroeconomics.\n\n" -"group": "mmlu_alt_ov_04c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_mathematics.yaml deleted file mode 100644 index e99c141b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_mathematics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school mathematics.\n\n" -"group": "mmlu_alt_ov_04c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_microeconomics.yaml deleted file mode 100644 index 8173caaa..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_microeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_microeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school microeconomics.\n\n" -"group": "mmlu_alt_ov_04c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_physics.yaml deleted file mode 100644 index 0060251f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_physics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school physics.\n\n" -"group": "mmlu_alt_ov_04c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_high_school_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_psychology.yaml deleted file mode 100644 index 5ad89e95..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_psychology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school psychology.\n\n" -"group": "mmlu_alt_ov_04c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_statistics.yaml deleted file mode 100644 index ebe72eac..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_statistics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_statistics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school statistics.\n\n" -"group": "mmlu_alt_ov_04c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_us_history.yaml deleted file mode 100644 index 9ba0a18a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_us_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_us_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school us history.\n\n" -"group": "mmlu_alt_ov_04c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_world_history.yaml deleted file mode 100644 index e634f4af..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_high_school_world_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_world_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school world history.\n\n" -"group": "mmlu_alt_ov_04c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_human_aging.yaml deleted file mode 100644 index 9510ba38..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_human_aging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_aging" -"description": "The following are multiple choice questions (with answers) about human\ - \ aging.\n\n" -"group": "mmlu_alt_ov_04c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_human_aging" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_human_sexuality.yaml deleted file mode 100644 index 287bbbd7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_human_sexuality.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_sexuality" -"description": "The following are multiple choice questions (with answers) about human\ - \ sexuality.\n\n" -"group": "mmlu_alt_ov_04c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_human_sexuality" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_international_law.yaml deleted file mode 100644 index 3c4b4e77..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_international_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "international_law" -"description": "The following are multiple choice questions (with answers) about international\ - \ law.\n\n" -"group": "mmlu_alt_ov_04c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_international_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_jurisprudence.yaml deleted file mode 100644 index b3a269b0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_jurisprudence.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "jurisprudence" -"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ - \n" -"group": "mmlu_alt_ov_04c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_jurisprudence" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_logical_fallacies.yaml deleted file mode 100644 index 54a2680f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_logical_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "logical_fallacies" -"description": "The following are multiple choice questions (with answers) about logical\ - \ fallacies.\n\n" -"group": "mmlu_alt_ov_04c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_machine_learning.yaml deleted file mode 100644 index 708c0c80..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_machine_learning.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "machine_learning" -"description": "The following are multiple choice questions (with answers) about machine\ - \ learning.\n\n" -"group": "mmlu_alt_ov_04c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_machine_learning" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_management.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_management.yaml deleted file mode 100644 index 7467c45e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_management.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "management" -"description": "The following are multiple choice questions (with answers) about management.\n\ - \n" -"group": "mmlu_alt_ov_04c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_management" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_marketing.yaml deleted file mode 100644 index 3567ae8d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_marketing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "marketing" -"description": "The following are multiple choice questions (with answers) about marketing.\n\ - \n" -"group": "mmlu_alt_ov_04c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_marketing" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_medical_genetics.yaml deleted file mode 100644 index 3f9686c1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_medical_genetics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "medical_genetics" -"description": "The following are multiple choice questions (with answers) about medical\ - \ genetics.\n\n" -"group": "mmlu_alt_ov_04c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_medical_genetics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_miscellaneous.yaml deleted file mode 100644 index 4529c5ea..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_miscellaneous.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "miscellaneous" -"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ - \n" -"group": "mmlu_alt_ov_04c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_miscellaneous" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_moral_disputes.yaml deleted file mode 100644 index 3f6f4da3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_moral_disputes.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_disputes" -"description": "The following are multiple choice questions (with answers) about moral\ - \ disputes.\n\n" -"group": "mmlu_alt_ov_04c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_moral_disputes" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_moral_scenarios.yaml deleted file mode 100644 index 1b5a23b6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_moral_scenarios.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_scenarios" -"description": "The following are multiple choice questions (with answers) about moral\ - \ scenarios.\n\n" -"group": "mmlu_alt_ov_04c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_nutrition.yaml deleted file mode 100644 index b3697fdb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_nutrition.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "nutrition" -"description": "The following are multiple choice questions (with answers) about nutrition.\n\ - \n" -"group": "mmlu_alt_ov_04c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_nutrition" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_philosophy.yaml deleted file mode 100644 index fee89c7a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_philosophy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "philosophy" -"description": "The following are multiple choice questions (with answers) about philosophy.\n\ - \n" -"group": "mmlu_alt_ov_04c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_philosophy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_prehistory.yaml deleted file mode 100644 index 3b9ebc08..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_prehistory.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "prehistory" -"description": "The following are multiple choice questions (with answers) about prehistory.\n\ - \n" -"group": "mmlu_alt_ov_04c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_prehistory" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_professional_accounting.yaml deleted file mode 100644 index 36419277..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_professional_accounting.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_accounting" -"description": "The following are multiple choice questions (with answers) about professional\ - \ accounting.\n\n" -"group": "mmlu_alt_ov_04c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_professional_accounting" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_professional_law.yaml deleted file mode 100644 index a40f78c4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_professional_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_law" -"description": "The following are multiple choice questions (with answers) about professional\ - \ law.\n\n" -"group": "mmlu_alt_ov_04c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_professional_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_professional_medicine.yaml deleted file mode 100644 index 149272c6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_professional_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_medicine" -"description": "The following are multiple choice questions (with answers) about professional\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_04c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_professional_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_professional_psychology.yaml deleted file mode 100644 index f208377a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_professional_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_psychology" -"description": "The following are multiple choice questions (with answers) about professional\ - \ psychology.\n\n" -"group": "mmlu_alt_ov_04c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_professional_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_public_relations.yaml deleted file mode 100644 index ed5b5314..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_public_relations.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "public_relations" -"description": "The following are multiple choice questions (with answers) about public\ - \ relations.\n\n" -"group": "mmlu_alt_ov_04c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_public_relations" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_security_studies.yaml deleted file mode 100644 index 53d538b4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_security_studies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "security_studies" -"description": "The following are multiple choice questions (with answers) about security\ - \ studies.\n\n" -"group": "mmlu_alt_ov_04c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_security_studies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_sociology.yaml deleted file mode 100644 index 2ccd329f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_sociology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sociology" -"description": "The following are multiple choice questions (with answers) about sociology.\n\ - \n" -"group": "mmlu_alt_ov_04c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_sociology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_us_foreign_policy.yaml deleted file mode 100644 index 2d29fb76..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_us_foreign_policy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "us_foreign_policy" -"description": "The following are multiple choice questions (with answers) about us\ - \ foreign policy.\n\n" -"group": "mmlu_alt_ov_04c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_virology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_virology.yaml deleted file mode 100644 index ea188a0d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_virology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "virology" -"description": "The following are multiple choice questions (with answers) about virology.\n\ - \n" -"group": "mmlu_alt_ov_04c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_virology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_world_religions.yaml deleted file mode 100644 index 4ee86870..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_04/c/mmlu_world_religions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "world_religions" -"description": "The following are multiple choice questions (with answers) about world\ - \ religions.\n\n" -"group": "mmlu_alt_ov_04c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_04c_world_religions" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/_mmlu_ov_05.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/_mmlu_ov_05.yaml deleted file mode 100644 index da5f4d3c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/_mmlu_ov_05.yaml +++ /dev/null @@ -1,5 +0,0 @@ -group: mmlu_alt_ov_05 -task: - - mmlu_alt_ov_05a - - mmlu_alt_ov_05b - - mmlu_alt_ov_05c diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/_mmlu.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/_mmlu.yaml deleted file mode 100644 index f70d92f3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/_mmlu.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: mmlu_alt_ov_05a -task: - - mmlu_alt_ov_05a_stem - - mmlu_alt_ov_05a_other - - mmlu_alt_ov_05a_social_sciences - - mmlu_alt_ov_05a_humanities diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/_template_yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/_template_yaml deleted file mode 100644 index 2382e8df..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/_template_yaml +++ /dev/null @@ -1,13 +0,0 @@ -dataset_path: hails/mmlu_no_train -test_split: test -fewshot_split: dev -fewshot_config: - sampler: first_n -output_type: multiple_choice -doc_to_text: !function ../../../styles.template_05 -doc_to_choice: !function ../../../styles.choice_05a -doc_to_target: answer -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_abstract_algebra.yaml deleted file mode 100644 index 5bbb4d49..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_abstract_algebra.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "abstract_algebra" -"description": "The following are multiple choice questions (with answers) about abstract\ - \ algebra.\n\n" -"group": "mmlu_alt_ov_05a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_anatomy.yaml deleted file mode 100644 index cb410f59..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_anatomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "anatomy" -"description": "The following are multiple choice questions (with answers) about anatomy.\n\ - \n" -"group": "mmlu_alt_ov_05a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_anatomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_astronomy.yaml deleted file mode 100644 index 3d29c90d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_astronomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "astronomy" -"description": "The following are multiple choice questions (with answers) about astronomy.\n\ - \n" -"group": "mmlu_alt_ov_05a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_astronomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_business_ethics.yaml deleted file mode 100644 index a6f32e9a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_business_ethics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "business_ethics" -"description": "The following are multiple choice questions (with answers) about business\ - \ ethics.\n\n" -"group": "mmlu_alt_ov_05a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_business_ethics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_clinical_knowledge.yaml deleted file mode 100644 index aac82a50..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_clinical_knowledge.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "clinical_knowledge" -"description": "The following are multiple choice questions (with answers) about clinical\ - \ knowledge.\n\n" -"group": "mmlu_alt_ov_05a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_college_biology.yaml deleted file mode 100644 index 7b2e0958..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_college_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_biology" -"description": "The following are multiple choice questions (with answers) about college\ - \ biology.\n\n" -"group": "mmlu_alt_ov_05a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_college_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_college_chemistry.yaml deleted file mode 100644 index 1695527b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_college_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_chemistry" -"description": "The following are multiple choice questions (with answers) about college\ - \ chemistry.\n\n" -"group": "mmlu_alt_ov_05a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_college_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_college_computer_science.yaml deleted file mode 100644 index f0f45f3b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_college_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_computer_science" -"description": "The following are multiple choice questions (with answers) about college\ - \ computer science.\n\n" -"group": "mmlu_alt_ov_05a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_college_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_college_mathematics.yaml deleted file mode 100644 index b65992b1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_college_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_mathematics" -"description": "The following are multiple choice questions (with answers) about college\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_05a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_college_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_college_medicine.yaml deleted file mode 100644 index 7167bb76..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_college_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_medicine" -"description": "The following are multiple choice questions (with answers) about college\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_05a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_college_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_college_physics.yaml deleted file mode 100644 index a5833f09..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_college_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_physics" -"description": "The following are multiple choice questions (with answers) about college\ - \ physics.\n\n" -"group": "mmlu_alt_ov_05a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_college_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_computer_security.yaml deleted file mode 100644 index c656c893..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_computer_security.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "computer_security" -"description": "The following are multiple choice questions (with answers) about computer\ - \ security.\n\n" -"group": "mmlu_alt_ov_05a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_computer_security" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_conceptual_physics.yaml deleted file mode 100644 index a765f164..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_conceptual_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "conceptual_physics" -"description": "The following are multiple choice questions (with answers) about conceptual\ - \ physics.\n\n" -"group": "mmlu_alt_ov_05a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_econometrics.yaml deleted file mode 100644 index 022c1e46..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_econometrics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "econometrics" -"description": "The following are multiple choice questions (with answers) about econometrics.\n\ - \n" -"group": "mmlu_alt_ov_05a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_econometrics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_electrical_engineering.yaml deleted file mode 100644 index 417cb37d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_electrical_engineering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "electrical_engineering" -"description": "The following are multiple choice questions (with answers) about electrical\ - \ engineering.\n\n" -"group": "mmlu_alt_ov_05a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_elementary_mathematics.yaml deleted file mode 100644 index 89ae4af9..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_elementary_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "elementary_mathematics" -"description": "The following are multiple choice questions (with answers) about elementary\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_05a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_formal_logic.yaml deleted file mode 100644 index aa186cd6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_formal_logic.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_logic" -"description": "The following are multiple choice questions (with answers) about formal\ - \ logic.\n\n" -"group": "mmlu_alt_ov_05a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_formal_logic" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_global_facts.yaml deleted file mode 100644 index 9c3505e3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_global_facts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "global_facts" -"description": "The following are multiple choice questions (with answers) about global\ - \ facts.\n\n" -"group": "mmlu_alt_ov_05a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_global_facts" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_biology.yaml deleted file mode 100644 index 61452db3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_biology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school biology.\n\n" -"group": "mmlu_alt_ov_05a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_high_school_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_chemistry.yaml deleted file mode 100644 index 707280da..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_chemistry" -"description": "The following are multiple choice questions (with answers) about high\ - \ school chemistry.\n\n" -"group": "mmlu_alt_ov_05a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_computer_science.yaml deleted file mode 100644 index 94e0a579..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_computer_science" -"description": "The following are multiple choice questions (with answers) about high\ - \ school computer science.\n\n" -"group": "mmlu_alt_ov_05a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_european_history.yaml deleted file mode 100644 index 9b199ea5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_european_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_european_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school european history.\n\n" -"group": "mmlu_alt_ov_05a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_geography.yaml deleted file mode 100644 index 9d7bce73..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_geography.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_geography" -"description": "The following are multiple choice questions (with answers) about high\ - \ school geography.\n\n" -"group": "mmlu_alt_ov_05a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_high_school_geography" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_government_and_politics.yaml deleted file mode 100644 index c629273e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_government_and_politics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_government_and_politics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school government and politics.\n\n" -"group": "mmlu_alt_ov_05a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_macroeconomics.yaml deleted file mode 100644 index 964ac9d3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_macroeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_macroeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school macroeconomics.\n\n" -"group": "mmlu_alt_ov_05a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_mathematics.yaml deleted file mode 100644 index 3304eec0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_mathematics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school mathematics.\n\n" -"group": "mmlu_alt_ov_05a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_microeconomics.yaml deleted file mode 100644 index 09b5d310..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_microeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_microeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school microeconomics.\n\n" -"group": "mmlu_alt_ov_05a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_physics.yaml deleted file mode 100644 index a26783e6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_physics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school physics.\n\n" -"group": "mmlu_alt_ov_05a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_high_school_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_psychology.yaml deleted file mode 100644 index 5b495c0b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_psychology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school psychology.\n\n" -"group": "mmlu_alt_ov_05a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_statistics.yaml deleted file mode 100644 index a27e82e2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_statistics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_statistics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school statistics.\n\n" -"group": "mmlu_alt_ov_05a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_us_history.yaml deleted file mode 100644 index f5eb66ff..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_us_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_us_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school us history.\n\n" -"group": "mmlu_alt_ov_05a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_world_history.yaml deleted file mode 100644 index 476aab63..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_high_school_world_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_world_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school world history.\n\n" -"group": "mmlu_alt_ov_05a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_human_aging.yaml deleted file mode 100644 index a0dc09bd..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_human_aging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_aging" -"description": "The following are multiple choice questions (with answers) about human\ - \ aging.\n\n" -"group": "mmlu_alt_ov_05a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_human_aging" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_human_sexuality.yaml deleted file mode 100644 index d2e9162d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_human_sexuality.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_sexuality" -"description": "The following are multiple choice questions (with answers) about human\ - \ sexuality.\n\n" -"group": "mmlu_alt_ov_05a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_human_sexuality" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_international_law.yaml deleted file mode 100644 index 69e41fea..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_international_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "international_law" -"description": "The following are multiple choice questions (with answers) about international\ - \ law.\n\n" -"group": "mmlu_alt_ov_05a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_international_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_jurisprudence.yaml deleted file mode 100644 index 7b98e5dc..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_jurisprudence.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "jurisprudence" -"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ - \n" -"group": "mmlu_alt_ov_05a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_jurisprudence" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_logical_fallacies.yaml deleted file mode 100644 index 2ef16cb9..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_logical_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "logical_fallacies" -"description": "The following are multiple choice questions (with answers) about logical\ - \ fallacies.\n\n" -"group": "mmlu_alt_ov_05a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_machine_learning.yaml deleted file mode 100644 index 49f2bda5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_machine_learning.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "machine_learning" -"description": "The following are multiple choice questions (with answers) about machine\ - \ learning.\n\n" -"group": "mmlu_alt_ov_05a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_machine_learning" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_management.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_management.yaml deleted file mode 100644 index 03412e62..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_management.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "management" -"description": "The following are multiple choice questions (with answers) about management.\n\ - \n" -"group": "mmlu_alt_ov_05a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_management" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_marketing.yaml deleted file mode 100644 index 7386ab1c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_marketing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "marketing" -"description": "The following are multiple choice questions (with answers) about marketing.\n\ - \n" -"group": "mmlu_alt_ov_05a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_marketing" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_medical_genetics.yaml deleted file mode 100644 index 506ff69b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_medical_genetics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "medical_genetics" -"description": "The following are multiple choice questions (with answers) about medical\ - \ genetics.\n\n" -"group": "mmlu_alt_ov_05a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_medical_genetics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_miscellaneous.yaml deleted file mode 100644 index 42466c93..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_miscellaneous.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "miscellaneous" -"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ - \n" -"group": "mmlu_alt_ov_05a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_miscellaneous" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_moral_disputes.yaml deleted file mode 100644 index e544a6e8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_moral_disputes.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_disputes" -"description": "The following are multiple choice questions (with answers) about moral\ - \ disputes.\n\n" -"group": "mmlu_alt_ov_05a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_moral_disputes" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_moral_scenarios.yaml deleted file mode 100644 index 31d66277..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_moral_scenarios.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_scenarios" -"description": "The following are multiple choice questions (with answers) about moral\ - \ scenarios.\n\n" -"group": "mmlu_alt_ov_05a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_nutrition.yaml deleted file mode 100644 index 4ef6da02..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_nutrition.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "nutrition" -"description": "The following are multiple choice questions (with answers) about nutrition.\n\ - \n" -"group": "mmlu_alt_ov_05a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_nutrition" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_philosophy.yaml deleted file mode 100644 index 44fb26b5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_philosophy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "philosophy" -"description": "The following are multiple choice questions (with answers) about philosophy.\n\ - \n" -"group": "mmlu_alt_ov_05a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_philosophy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_prehistory.yaml deleted file mode 100644 index a2eb9697..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_prehistory.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "prehistory" -"description": "The following are multiple choice questions (with answers) about prehistory.\n\ - \n" -"group": "mmlu_alt_ov_05a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_prehistory" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_professional_accounting.yaml deleted file mode 100644 index bd268b2b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_professional_accounting.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_accounting" -"description": "The following are multiple choice questions (with answers) about professional\ - \ accounting.\n\n" -"group": "mmlu_alt_ov_05a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_professional_accounting" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_professional_law.yaml deleted file mode 100644 index 3137eee9..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_professional_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_law" -"description": "The following are multiple choice questions (with answers) about professional\ - \ law.\n\n" -"group": "mmlu_alt_ov_05a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_professional_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_professional_medicine.yaml deleted file mode 100644 index 3533fde1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_professional_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_medicine" -"description": "The following are multiple choice questions (with answers) about professional\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_05a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_professional_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_professional_psychology.yaml deleted file mode 100644 index e5ac7be3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_professional_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_psychology" -"description": "The following are multiple choice questions (with answers) about professional\ - \ psychology.\n\n" -"group": "mmlu_alt_ov_05a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_professional_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_public_relations.yaml deleted file mode 100644 index 548bf1ca..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_public_relations.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "public_relations" -"description": "The following are multiple choice questions (with answers) about public\ - \ relations.\n\n" -"group": "mmlu_alt_ov_05a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_public_relations" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_security_studies.yaml deleted file mode 100644 index 07b08985..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_security_studies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "security_studies" -"description": "The following are multiple choice questions (with answers) about security\ - \ studies.\n\n" -"group": "mmlu_alt_ov_05a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_security_studies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_sociology.yaml deleted file mode 100644 index d70a37fb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_sociology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sociology" -"description": "The following are multiple choice questions (with answers) about sociology.\n\ - \n" -"group": "mmlu_alt_ov_05a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_sociology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_us_foreign_policy.yaml deleted file mode 100644 index 8c3161cb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_us_foreign_policy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "us_foreign_policy" -"description": "The following are multiple choice questions (with answers) about us\ - \ foreign policy.\n\n" -"group": "mmlu_alt_ov_05a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_virology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_virology.yaml deleted file mode 100644 index 1a037bcf..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_virology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "virology" -"description": "The following are multiple choice questions (with answers) about virology.\n\ - \n" -"group": "mmlu_alt_ov_05a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_virology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_world_religions.yaml deleted file mode 100644 index bcaad0cb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/a/mmlu_world_religions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "world_religions" -"description": "The following are multiple choice questions (with answers) about world\ - \ religions.\n\n" -"group": "mmlu_alt_ov_05a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_world_religions" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/_mmlu.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/_mmlu.yaml deleted file mode 100644 index ece20a94..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/_mmlu.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: mmlu_alt_ov_05b -task: - - mmlu_alt_ov_05b_stem - - mmlu_alt_ov_05b_other - - mmlu_alt_ov_05b_social_sciences - - mmlu_alt_ov_05b_humanities diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/_template_yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/_template_yaml deleted file mode 100644 index b14c6288..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/_template_yaml +++ /dev/null @@ -1,11 +0,0 @@ -dataset_path: hails/mmlu_no_train -test_split: test -fewshot_split: dev -output_type: multiple_choice -doc_to_text: !function ../../../styles.template_05 -doc_to_choice: !function ../../../styles.choice_05b -doc_to_target: answer -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_abstract_algebra.yaml deleted file mode 100644 index 10731bda..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_abstract_algebra.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "abstract_algebra" -"description": "The following are multiple choice questions (with answers) about abstract\ - \ algebra.\n\n" -"group": "mmlu_alt_ov_05b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_anatomy.yaml deleted file mode 100644 index a5bdcddf..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_anatomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "anatomy" -"description": "The following are multiple choice questions (with answers) about anatomy.\n\ - \n" -"group": "mmlu_alt_ov_05b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_anatomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_astronomy.yaml deleted file mode 100644 index 41d9a90d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_astronomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "astronomy" -"description": "The following are multiple choice questions (with answers) about astronomy.\n\ - \n" -"group": "mmlu_alt_ov_05b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_astronomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_business_ethics.yaml deleted file mode 100644 index 85e4117e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_business_ethics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "business_ethics" -"description": "The following are multiple choice questions (with answers) about business\ - \ ethics.\n\n" -"group": "mmlu_alt_ov_05b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_business_ethics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_clinical_knowledge.yaml deleted file mode 100644 index 6c905d18..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_clinical_knowledge.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "clinical_knowledge" -"description": "The following are multiple choice questions (with answers) about clinical\ - \ knowledge.\n\n" -"group": "mmlu_alt_ov_05b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_college_biology.yaml deleted file mode 100644 index 72a36377..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_college_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_biology" -"description": "The following are multiple choice questions (with answers) about college\ - \ biology.\n\n" -"group": "mmlu_alt_ov_05b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_college_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_college_chemistry.yaml deleted file mode 100644 index f29f8d0e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_college_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_chemistry" -"description": "The following are multiple choice questions (with answers) about college\ - \ chemistry.\n\n" -"group": "mmlu_alt_ov_05b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_college_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_college_computer_science.yaml deleted file mode 100644 index 916a25a8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_college_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_computer_science" -"description": "The following are multiple choice questions (with answers) about college\ - \ computer science.\n\n" -"group": "mmlu_alt_ov_05b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_college_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_college_mathematics.yaml deleted file mode 100644 index f9d1722c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_college_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_mathematics" -"description": "The following are multiple choice questions (with answers) about college\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_05b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_college_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_college_medicine.yaml deleted file mode 100644 index e95cfb00..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_college_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_medicine" -"description": "The following are multiple choice questions (with answers) about college\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_05b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_college_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_college_physics.yaml deleted file mode 100644 index 9b519110..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_college_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_physics" -"description": "The following are multiple choice questions (with answers) about college\ - \ physics.\n\n" -"group": "mmlu_alt_ov_05b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_college_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_computer_security.yaml deleted file mode 100644 index 77334a82..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_computer_security.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "computer_security" -"description": "The following are multiple choice questions (with answers) about computer\ - \ security.\n\n" -"group": "mmlu_alt_ov_05b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_computer_security" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_conceptual_physics.yaml deleted file mode 100644 index 77818214..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_conceptual_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "conceptual_physics" -"description": "The following are multiple choice questions (with answers) about conceptual\ - \ physics.\n\n" -"group": "mmlu_alt_ov_05b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_econometrics.yaml deleted file mode 100644 index 1aeb60f3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_econometrics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "econometrics" -"description": "The following are multiple choice questions (with answers) about econometrics.\n\ - \n" -"group": "mmlu_alt_ov_05b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_econometrics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_electrical_engineering.yaml deleted file mode 100644 index 58ed423b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_electrical_engineering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "electrical_engineering" -"description": "The following are multiple choice questions (with answers) about electrical\ - \ engineering.\n\n" -"group": "mmlu_alt_ov_05b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_elementary_mathematics.yaml deleted file mode 100644 index 548e0413..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_elementary_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "elementary_mathematics" -"description": "The following are multiple choice questions (with answers) about elementary\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_05b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_formal_logic.yaml deleted file mode 100644 index f07f5de8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_formal_logic.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_logic" -"description": "The following are multiple choice questions (with answers) about formal\ - \ logic.\n\n" -"group": "mmlu_alt_ov_05b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_formal_logic" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_global_facts.yaml deleted file mode 100644 index ec77b6e9..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_global_facts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "global_facts" -"description": "The following are multiple choice questions (with answers) about global\ - \ facts.\n\n" -"group": "mmlu_alt_ov_05b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_global_facts" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_biology.yaml deleted file mode 100644 index 85c228e5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_biology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school biology.\n\n" -"group": "mmlu_alt_ov_05b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_high_school_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_chemistry.yaml deleted file mode 100644 index 0e7f9ef6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_chemistry" -"description": "The following are multiple choice questions (with answers) about high\ - \ school chemistry.\n\n" -"group": "mmlu_alt_ov_05b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_computer_science.yaml deleted file mode 100644 index 07d40a6d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_computer_science" -"description": "The following are multiple choice questions (with answers) about high\ - \ school computer science.\n\n" -"group": "mmlu_alt_ov_05b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_european_history.yaml deleted file mode 100644 index 74051cc5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_european_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_european_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school european history.\n\n" -"group": "mmlu_alt_ov_05b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_geography.yaml deleted file mode 100644 index dfdda226..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_geography.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_geography" -"description": "The following are multiple choice questions (with answers) about high\ - \ school geography.\n\n" -"group": "mmlu_alt_ov_05b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_high_school_geography" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_government_and_politics.yaml deleted file mode 100644 index 30ee3766..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_government_and_politics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_government_and_politics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school government and politics.\n\n" -"group": "mmlu_alt_ov_05b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_macroeconomics.yaml deleted file mode 100644 index 3ac09e8c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_macroeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_macroeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school macroeconomics.\n\n" -"group": "mmlu_alt_ov_05b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_mathematics.yaml deleted file mode 100644 index 06ae5b54..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_mathematics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school mathematics.\n\n" -"group": "mmlu_alt_ov_05b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_microeconomics.yaml deleted file mode 100644 index 1565ba66..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_microeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_microeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school microeconomics.\n\n" -"group": "mmlu_alt_ov_05b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_physics.yaml deleted file mode 100644 index 4a07de6f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_physics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school physics.\n\n" -"group": "mmlu_alt_ov_05b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_high_school_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_psychology.yaml deleted file mode 100644 index 08cfc8bf..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_psychology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school psychology.\n\n" -"group": "mmlu_alt_ov_05b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_statistics.yaml deleted file mode 100644 index aaa099bc..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_statistics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_statistics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school statistics.\n\n" -"group": "mmlu_alt_ov_05b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_us_history.yaml deleted file mode 100644 index 29152b5d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_us_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_us_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school us history.\n\n" -"group": "mmlu_alt_ov_05b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_world_history.yaml deleted file mode 100644 index 54647dd0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_high_school_world_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_world_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school world history.\n\n" -"group": "mmlu_alt_ov_05b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_human_aging.yaml deleted file mode 100644 index 976621d2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_human_aging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_aging" -"description": "The following are multiple choice questions (with answers) about human\ - \ aging.\n\n" -"group": "mmlu_alt_ov_05b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_human_aging" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_human_sexuality.yaml deleted file mode 100644 index 1a4668c6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_human_sexuality.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_sexuality" -"description": "The following are multiple choice questions (with answers) about human\ - \ sexuality.\n\n" -"group": "mmlu_alt_ov_05b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_human_sexuality" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_international_law.yaml deleted file mode 100644 index 0b2626c3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_international_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "international_law" -"description": "The following are multiple choice questions (with answers) about international\ - \ law.\n\n" -"group": "mmlu_alt_ov_05b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_international_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_jurisprudence.yaml deleted file mode 100644 index d818f149..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_jurisprudence.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "jurisprudence" -"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ - \n" -"group": "mmlu_alt_ov_05b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_jurisprudence" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_logical_fallacies.yaml deleted file mode 100644 index 5eff9592..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_logical_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "logical_fallacies" -"description": "The following are multiple choice questions (with answers) about logical\ - \ fallacies.\n\n" -"group": "mmlu_alt_ov_05b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_machine_learning.yaml deleted file mode 100644 index 1bbdef46..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_machine_learning.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "machine_learning" -"description": "The following are multiple choice questions (with answers) about machine\ - \ learning.\n\n" -"group": "mmlu_alt_ov_05b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_machine_learning" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_management.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_management.yaml deleted file mode 100644 index ff536418..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_management.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "management" -"description": "The following are multiple choice questions (with answers) about management.\n\ - \n" -"group": "mmlu_alt_ov_05b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_management" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_marketing.yaml deleted file mode 100644 index cd07a402..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_marketing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "marketing" -"description": "The following are multiple choice questions (with answers) about marketing.\n\ - \n" -"group": "mmlu_alt_ov_05b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_marketing" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_medical_genetics.yaml deleted file mode 100644 index ec286d16..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_medical_genetics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "medical_genetics" -"description": "The following are multiple choice questions (with answers) about medical\ - \ genetics.\n\n" -"group": "mmlu_alt_ov_05b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_medical_genetics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_miscellaneous.yaml deleted file mode 100644 index 2d1bd3ca..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_miscellaneous.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "miscellaneous" -"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ - \n" -"group": "mmlu_alt_ov_05b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_miscellaneous" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_moral_disputes.yaml deleted file mode 100644 index c1f2fc79..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_moral_disputes.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_disputes" -"description": "The following are multiple choice questions (with answers) about moral\ - \ disputes.\n\n" -"group": "mmlu_alt_ov_05b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_moral_disputes" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_moral_scenarios.yaml deleted file mode 100644 index 4266c2a3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_moral_scenarios.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_scenarios" -"description": "The following are multiple choice questions (with answers) about moral\ - \ scenarios.\n\n" -"group": "mmlu_alt_ov_05b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_nutrition.yaml deleted file mode 100644 index 286e6c7b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_nutrition.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "nutrition" -"description": "The following are multiple choice questions (with answers) about nutrition.\n\ - \n" -"group": "mmlu_alt_ov_05b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_nutrition" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_philosophy.yaml deleted file mode 100644 index 84183735..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_philosophy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "philosophy" -"description": "The following are multiple choice questions (with answers) about philosophy.\n\ - \n" -"group": "mmlu_alt_ov_05b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_philosophy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_prehistory.yaml deleted file mode 100644 index 2b1b746f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_prehistory.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "prehistory" -"description": "The following are multiple choice questions (with answers) about prehistory.\n\ - \n" -"group": "mmlu_alt_ov_05b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_prehistory" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_professional_accounting.yaml deleted file mode 100644 index d61861e9..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_professional_accounting.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_accounting" -"description": "The following are multiple choice questions (with answers) about professional\ - \ accounting.\n\n" -"group": "mmlu_alt_ov_05b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_professional_accounting" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_professional_law.yaml deleted file mode 100644 index 8c056a95..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_professional_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_law" -"description": "The following are multiple choice questions (with answers) about professional\ - \ law.\n\n" -"group": "mmlu_alt_ov_05b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_professional_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_professional_medicine.yaml deleted file mode 100644 index af36b4dd..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_professional_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_medicine" -"description": "The following are multiple choice questions (with answers) about professional\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_05b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_professional_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_professional_psychology.yaml deleted file mode 100644 index 8cdec089..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_professional_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_psychology" -"description": "The following are multiple choice questions (with answers) about professional\ - \ psychology.\n\n" -"group": "mmlu_alt_ov_05b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_professional_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_public_relations.yaml deleted file mode 100644 index 30882bb2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_public_relations.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "public_relations" -"description": "The following are multiple choice questions (with answers) about public\ - \ relations.\n\n" -"group": "mmlu_alt_ov_05b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_public_relations" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_security_studies.yaml deleted file mode 100644 index 92b73a2a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_security_studies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "security_studies" -"description": "The following are multiple choice questions (with answers) about security\ - \ studies.\n\n" -"group": "mmlu_alt_ov_05b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_security_studies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_sociology.yaml deleted file mode 100644 index 727ae1a8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_sociology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sociology" -"description": "The following are multiple choice questions (with answers) about sociology.\n\ - \n" -"group": "mmlu_alt_ov_05b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_sociology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_us_foreign_policy.yaml deleted file mode 100644 index 4685b4a8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_us_foreign_policy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "us_foreign_policy" -"description": "The following are multiple choice questions (with answers) about us\ - \ foreign policy.\n\n" -"group": "mmlu_alt_ov_05b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_virology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_virology.yaml deleted file mode 100644 index 8c0e69da..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_virology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "virology" -"description": "The following are multiple choice questions (with answers) about virology.\n\ - \n" -"group": "mmlu_alt_ov_05b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_virology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_world_religions.yaml deleted file mode 100644 index d85a0006..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/b/mmlu_world_religions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "world_religions" -"description": "The following are multiple choice questions (with answers) about world\ - \ religions.\n\n" -"group": "mmlu_alt_ov_05b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05b_world_religions" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/_mmlu.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/_mmlu.yaml deleted file mode 100644 index 88041f27..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/_mmlu.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: mmlu_alt_ov_05c -task: - - mmlu_alt_ov_05c_stem - - mmlu_alt_ov_05c_other - - mmlu_alt_ov_05c_social_sciences - - mmlu_alt_ov_05c_humanities diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/_template_yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/_template_yaml deleted file mode 100644 index 82259f03..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/_template_yaml +++ /dev/null @@ -1,11 +0,0 @@ -dataset_path: hails/mmlu_no_train -test_split: test -fewshot_split: dev -output_type: multiple_choice -doc_to_text: !function ../../../styles.template_05 -doc_to_choice: !function ../../../styles.choice_05c -doc_to_target: answer -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_abstract_algebra.yaml deleted file mode 100644 index 4d6b10f8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_abstract_algebra.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "abstract_algebra" -"description": "The following are multiple choice questions (with answers) about abstract\ - \ algebra.\n\n" -"group": "mmlu_alt_ov_05c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_anatomy.yaml deleted file mode 100644 index 5a04c43d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_anatomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "anatomy" -"description": "The following are multiple choice questions (with answers) about anatomy.\n\ - \n" -"group": "mmlu_alt_ov_05c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_anatomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_astronomy.yaml deleted file mode 100644 index f925929f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_astronomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "astronomy" -"description": "The following are multiple choice questions (with answers) about astronomy.\n\ - \n" -"group": "mmlu_alt_ov_05c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_astronomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_business_ethics.yaml deleted file mode 100644 index 1bff1ab0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_business_ethics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "business_ethics" -"description": "The following are multiple choice questions (with answers) about business\ - \ ethics.\n\n" -"group": "mmlu_alt_ov_05c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_business_ethics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_clinical_knowledge.yaml deleted file mode 100644 index abeee50a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_clinical_knowledge.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "clinical_knowledge" -"description": "The following are multiple choice questions (with answers) about clinical\ - \ knowledge.\n\n" -"group": "mmlu_alt_ov_05c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_college_biology.yaml deleted file mode 100644 index d85f4238..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_college_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_biology" -"description": "The following are multiple choice questions (with answers) about college\ - \ biology.\n\n" -"group": "mmlu_alt_ov_05c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_college_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_college_chemistry.yaml deleted file mode 100644 index fd2f672f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_college_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_chemistry" -"description": "The following are multiple choice questions (with answers) about college\ - \ chemistry.\n\n" -"group": "mmlu_alt_ov_05c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_college_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_college_computer_science.yaml deleted file mode 100644 index 02d07d5f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_college_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_computer_science" -"description": "The following are multiple choice questions (with answers) about college\ - \ computer science.\n\n" -"group": "mmlu_alt_ov_05c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_college_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_college_mathematics.yaml deleted file mode 100644 index 0e18b953..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_college_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_mathematics" -"description": "The following are multiple choice questions (with answers) about college\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_05c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_college_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_college_medicine.yaml deleted file mode 100644 index d68d7a3d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_college_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_medicine" -"description": "The following are multiple choice questions (with answers) about college\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_05c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_college_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_college_physics.yaml deleted file mode 100644 index f200a990..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_college_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_physics" -"description": "The following are multiple choice questions (with answers) about college\ - \ physics.\n\n" -"group": "mmlu_alt_ov_05c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_college_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_computer_security.yaml deleted file mode 100644 index aa926444..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_computer_security.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "computer_security" -"description": "The following are multiple choice questions (with answers) about computer\ - \ security.\n\n" -"group": "mmlu_alt_ov_05c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_computer_security" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_conceptual_physics.yaml deleted file mode 100644 index 7cb1151f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_conceptual_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "conceptual_physics" -"description": "The following are multiple choice questions (with answers) about conceptual\ - \ physics.\n\n" -"group": "mmlu_alt_ov_05c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_econometrics.yaml deleted file mode 100644 index 6253cb57..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_econometrics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "econometrics" -"description": "The following are multiple choice questions (with answers) about econometrics.\n\ - \n" -"group": "mmlu_alt_ov_05c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_econometrics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_electrical_engineering.yaml deleted file mode 100644 index f75f1037..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_electrical_engineering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "electrical_engineering" -"description": "The following are multiple choice questions (with answers) about electrical\ - \ engineering.\n\n" -"group": "mmlu_alt_ov_05c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_elementary_mathematics.yaml deleted file mode 100644 index f19f9904..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_elementary_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "elementary_mathematics" -"description": "The following are multiple choice questions (with answers) about elementary\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_05c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_formal_logic.yaml deleted file mode 100644 index a8b8c1d9..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_formal_logic.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_logic" -"description": "The following are multiple choice questions (with answers) about formal\ - \ logic.\n\n" -"group": "mmlu_alt_ov_05c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_formal_logic" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_global_facts.yaml deleted file mode 100644 index 85993966..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_global_facts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "global_facts" -"description": "The following are multiple choice questions (with answers) about global\ - \ facts.\n\n" -"group": "mmlu_alt_ov_05c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_global_facts" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_biology.yaml deleted file mode 100644 index b3ba98ab..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_biology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school biology.\n\n" -"group": "mmlu_alt_ov_05c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_high_school_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_chemistry.yaml deleted file mode 100644 index 7f25e96b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_chemistry" -"description": "The following are multiple choice questions (with answers) about high\ - \ school chemistry.\n\n" -"group": "mmlu_alt_ov_05c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_computer_science.yaml deleted file mode 100644 index a715a00e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_computer_science" -"description": "The following are multiple choice questions (with answers) about high\ - \ school computer science.\n\n" -"group": "mmlu_alt_ov_05c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_european_history.yaml deleted file mode 100644 index 6e9921ee..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_european_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_european_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school european history.\n\n" -"group": "mmlu_alt_ov_05c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_geography.yaml deleted file mode 100644 index 3523e372..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_geography.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_geography" -"description": "The following are multiple choice questions (with answers) about high\ - \ school geography.\n\n" -"group": "mmlu_alt_ov_05c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_high_school_geography" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_government_and_politics.yaml deleted file mode 100644 index 62010d86..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_government_and_politics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_government_and_politics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school government and politics.\n\n" -"group": "mmlu_alt_ov_05c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_macroeconomics.yaml deleted file mode 100644 index 1b1c566a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_macroeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_macroeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school macroeconomics.\n\n" -"group": "mmlu_alt_ov_05c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_mathematics.yaml deleted file mode 100644 index db8370a4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_mathematics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school mathematics.\n\n" -"group": "mmlu_alt_ov_05c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_microeconomics.yaml deleted file mode 100644 index 9b2c003c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_microeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_microeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school microeconomics.\n\n" -"group": "mmlu_alt_ov_05c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_physics.yaml deleted file mode 100644 index 697d7af6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_physics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school physics.\n\n" -"group": "mmlu_alt_ov_05c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_high_school_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_psychology.yaml deleted file mode 100644 index e758a3a9..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_psychology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school psychology.\n\n" -"group": "mmlu_alt_ov_05c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_statistics.yaml deleted file mode 100644 index 93b2f70d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_statistics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_statistics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school statistics.\n\n" -"group": "mmlu_alt_ov_05c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_us_history.yaml deleted file mode 100644 index a30146a0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_us_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_us_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school us history.\n\n" -"group": "mmlu_alt_ov_05c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_world_history.yaml deleted file mode 100644 index b3868797..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_high_school_world_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_world_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school world history.\n\n" -"group": "mmlu_alt_ov_05c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_human_aging.yaml deleted file mode 100644 index 0b3a0478..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_human_aging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_aging" -"description": "The following are multiple choice questions (with answers) about human\ - \ aging.\n\n" -"group": "mmlu_alt_ov_05c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_human_aging" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_human_sexuality.yaml deleted file mode 100644 index d425bcc2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_human_sexuality.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_sexuality" -"description": "The following are multiple choice questions (with answers) about human\ - \ sexuality.\n\n" -"group": "mmlu_alt_ov_05c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_human_sexuality" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_international_law.yaml deleted file mode 100644 index 13b5a716..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_international_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "international_law" -"description": "The following are multiple choice questions (with answers) about international\ - \ law.\n\n" -"group": "mmlu_alt_ov_05c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_international_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_jurisprudence.yaml deleted file mode 100644 index 8bbf9823..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_jurisprudence.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "jurisprudence" -"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ - \n" -"group": "mmlu_alt_ov_05c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_jurisprudence" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_logical_fallacies.yaml deleted file mode 100644 index 315af4b4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_logical_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "logical_fallacies" -"description": "The following are multiple choice questions (with answers) about logical\ - \ fallacies.\n\n" -"group": "mmlu_alt_ov_05c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_machine_learning.yaml deleted file mode 100644 index ab99a814..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_machine_learning.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "machine_learning" -"description": "The following are multiple choice questions (with answers) about machine\ - \ learning.\n\n" -"group": "mmlu_alt_ov_05c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_machine_learning" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_management.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_management.yaml deleted file mode 100644 index 16715d6d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_management.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "management" -"description": "The following are multiple choice questions (with answers) about management.\n\ - \n" -"group": "mmlu_alt_ov_05c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_management" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_marketing.yaml deleted file mode 100644 index c31dba8a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_marketing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "marketing" -"description": "The following are multiple choice questions (with answers) about marketing.\n\ - \n" -"group": "mmlu_alt_ov_05c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_marketing" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_medical_genetics.yaml deleted file mode 100644 index bc136da9..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_medical_genetics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "medical_genetics" -"description": "The following are multiple choice questions (with answers) about medical\ - \ genetics.\n\n" -"group": "mmlu_alt_ov_05c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_medical_genetics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_miscellaneous.yaml deleted file mode 100644 index 3ad9a519..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_miscellaneous.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "miscellaneous" -"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ - \n" -"group": "mmlu_alt_ov_05c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_miscellaneous" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_moral_disputes.yaml deleted file mode 100644 index ef20ece5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_moral_disputes.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_disputes" -"description": "The following are multiple choice questions (with answers) about moral\ - \ disputes.\n\n" -"group": "mmlu_alt_ov_05c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_moral_disputes" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_moral_scenarios.yaml deleted file mode 100644 index 83d2ebbe..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_moral_scenarios.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_scenarios" -"description": "The following are multiple choice questions (with answers) about moral\ - \ scenarios.\n\n" -"group": "mmlu_alt_ov_05c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_nutrition.yaml deleted file mode 100644 index 2cfb0845..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_nutrition.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "nutrition" -"description": "The following are multiple choice questions (with answers) about nutrition.\n\ - \n" -"group": "mmlu_alt_ov_05c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_nutrition" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_philosophy.yaml deleted file mode 100644 index 9e015ff1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_philosophy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "philosophy" -"description": "The following are multiple choice questions (with answers) about philosophy.\n\ - \n" -"group": "mmlu_alt_ov_05c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_philosophy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_prehistory.yaml deleted file mode 100644 index 9656cfe5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_prehistory.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "prehistory" -"description": "The following are multiple choice questions (with answers) about prehistory.\n\ - \n" -"group": "mmlu_alt_ov_05c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_prehistory" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_professional_accounting.yaml deleted file mode 100644 index 4ecbe7cf..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_professional_accounting.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_accounting" -"description": "The following are multiple choice questions (with answers) about professional\ - \ accounting.\n\n" -"group": "mmlu_alt_ov_05c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_professional_accounting" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_professional_law.yaml deleted file mode 100644 index 355bbdb7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_professional_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_law" -"description": "The following are multiple choice questions (with answers) about professional\ - \ law.\n\n" -"group": "mmlu_alt_ov_05c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_professional_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_professional_medicine.yaml deleted file mode 100644 index c20f3c71..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_professional_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_medicine" -"description": "The following are multiple choice questions (with answers) about professional\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_05c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_professional_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_professional_psychology.yaml deleted file mode 100644 index 54cdb1ae..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_professional_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_psychology" -"description": "The following are multiple choice questions (with answers) about professional\ - \ psychology.\n\n" -"group": "mmlu_alt_ov_05c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_professional_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_public_relations.yaml deleted file mode 100644 index 8a8123c2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_public_relations.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "public_relations" -"description": "The following are multiple choice questions (with answers) about public\ - \ relations.\n\n" -"group": "mmlu_alt_ov_05c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_public_relations" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_security_studies.yaml deleted file mode 100644 index 8ed204c1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_security_studies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "security_studies" -"description": "The following are multiple choice questions (with answers) about security\ - \ studies.\n\n" -"group": "mmlu_alt_ov_05c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_security_studies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_sociology.yaml deleted file mode 100644 index 24f6ac68..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_sociology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sociology" -"description": "The following are multiple choice questions (with answers) about sociology.\n\ - \n" -"group": "mmlu_alt_ov_05c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_sociology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_us_foreign_policy.yaml deleted file mode 100644 index 027f9842..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_us_foreign_policy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "us_foreign_policy" -"description": "The following are multiple choice questions (with answers) about us\ - \ foreign policy.\n\n" -"group": "mmlu_alt_ov_05c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_virology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_virology.yaml deleted file mode 100644 index d33a4fef..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_virology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "virology" -"description": "The following are multiple choice questions (with answers) about virology.\n\ - \n" -"group": "mmlu_alt_ov_05c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_virology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_world_religions.yaml deleted file mode 100644 index 14d1d836..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05/c/mmlu_world_religions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "world_religions" -"description": "The following are multiple choice questions (with answers) about world\ - \ religions.\n\n" -"group": "mmlu_alt_ov_05c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05c_world_religions" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/_mmlu.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/_mmlu.yaml deleted file mode 100644 index 0782fefc..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/_mmlu.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: mmlu_alt_ov_05a_generative -task: - - mmlu_alt_ov_05a_stem_generative - - mmlu_alt_ov_05a_other_generative - - mmlu_alt_ov_05a_social_sciences_generative - - mmlu_alt_ov_05a_humanities_generative diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/_template_yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/_template_yaml deleted file mode 100644 index 11ed09b7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/_template_yaml +++ /dev/null @@ -1,18 +0,0 @@ -dataset_path: hails/mmlu_no_train -test_split: test -fewshot_split: dev -fewshot_config: - sampler: first_n -output_type: generate_until -doc_to_text: !function ../../../styles.template_05 -doc_to_target: !function ../../../styles.target_05a -generation_kwargs: - until: - - "" - - "<0x0A>" -metric_list: - - metric: exact_match - aggregation: mean - higher_is_better: true -metadata: - version: 0.0 diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_abstract_algebra.yaml deleted file mode 100644 index fcc57e75..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_abstract_algebra.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "abstract_algebra" -"description": "The following are multiple choice questions (with answers) about abstract\ - \ algebra.\n\n" -"group": "mmlu_alt_ov_05a_stem_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_abstract_algebra_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_anatomy.yaml deleted file mode 100644 index b3faed6c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_anatomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "anatomy" -"description": "The following are multiple choice questions (with answers) about anatomy.\n\ - \n" -"group": "mmlu_alt_ov_05a_stem_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_anatomy_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_astronomy.yaml deleted file mode 100644 index 795e3fe3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_astronomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "astronomy" -"description": "The following are multiple choice questions (with answers) about astronomy.\n\ - \n" -"group": "mmlu_alt_ov_05a_stem_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_astronomy_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_business_ethics.yaml deleted file mode 100644 index 07c49e21..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_business_ethics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "business_ethics" -"description": "The following are multiple choice questions (with answers) about business\ - \ ethics.\n\n" -"group": "mmlu_alt_ov_05a_other_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_business_ethics_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_clinical_knowledge.yaml deleted file mode 100644 index 11a7c7f9..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_clinical_knowledge.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "clinical_knowledge" -"description": "The following are multiple choice questions (with answers) about clinical\ - \ knowledge.\n\n" -"group": "mmlu_alt_ov_05a_other_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_clinical_knowledge_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_college_biology.yaml deleted file mode 100644 index ed721c0d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_college_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_biology" -"description": "The following are multiple choice questions (with answers) about college\ - \ biology.\n\n" -"group": "mmlu_alt_ov_05a_stem_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_college_biology_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_college_chemistry.yaml deleted file mode 100644 index e5e2246c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_college_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_chemistry" -"description": "The following are multiple choice questions (with answers) about college\ - \ chemistry.\n\n" -"group": "mmlu_alt_ov_05a_stem_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_college_chemistry_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_college_computer_science.yaml deleted file mode 100644 index 6fc02bfa..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_college_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_computer_science" -"description": "The following are multiple choice questions (with answers) about college\ - \ computer science.\n\n" -"group": "mmlu_alt_ov_05a_stem_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_college_computer_science_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_college_mathematics.yaml deleted file mode 100644 index 6b55aece..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_college_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_mathematics" -"description": "The following are multiple choice questions (with answers) about college\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_05a_stem_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_college_mathematics_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_college_medicine.yaml deleted file mode 100644 index db197fac..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_college_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_medicine" -"description": "The following are multiple choice questions (with answers) about college\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_05a_other_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_college_medicine_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_college_physics.yaml deleted file mode 100644 index 61d4e99a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_college_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_physics" -"description": "The following are multiple choice questions (with answers) about college\ - \ physics.\n\n" -"group": "mmlu_alt_ov_05a_stem_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_college_physics_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_computer_security.yaml deleted file mode 100644 index ff8183e7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_computer_security.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "computer_security" -"description": "The following are multiple choice questions (with answers) about computer\ - \ security.\n\n" -"group": "mmlu_alt_ov_05a_stem_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_computer_security_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_conceptual_physics.yaml deleted file mode 100644 index 6c4d3a82..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_conceptual_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "conceptual_physics" -"description": "The following are multiple choice questions (with answers) about conceptual\ - \ physics.\n\n" -"group": "mmlu_alt_ov_05a_stem_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_conceptual_physics_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_econometrics.yaml deleted file mode 100644 index 16a94e0a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_econometrics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "econometrics" -"description": "The following are multiple choice questions (with answers) about econometrics.\n\ - \n" -"group": "mmlu_alt_ov_05a_social_sciences_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_econometrics_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_electrical_engineering.yaml deleted file mode 100644 index d0c1d37f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_electrical_engineering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "electrical_engineering" -"description": "The following are multiple choice questions (with answers) about electrical\ - \ engineering.\n\n" -"group": "mmlu_alt_ov_05a_stem_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_electrical_engineering_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_elementary_mathematics.yaml deleted file mode 100644 index 4b662758..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_elementary_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "elementary_mathematics" -"description": "The following are multiple choice questions (with answers) about elementary\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_05a_stem_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_elementary_mathematics_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_formal_logic.yaml deleted file mode 100644 index 612d7f61..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_formal_logic.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_logic" -"description": "The following are multiple choice questions (with answers) about formal\ - \ logic.\n\n" -"group": "mmlu_alt_ov_05a_humanities_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_formal_logic_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_global_facts.yaml deleted file mode 100644 index 316f0025..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_global_facts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "global_facts" -"description": "The following are multiple choice questions (with answers) about global\ - \ facts.\n\n" -"group": "mmlu_alt_ov_05a_other_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_global_facts_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_biology.yaml deleted file mode 100644 index df743be3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_biology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school biology.\n\n" -"group": "mmlu_alt_ov_05a_stem_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_high_school_biology_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_chemistry.yaml deleted file mode 100644 index 89100d15..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_chemistry" -"description": "The following are multiple choice questions (with answers) about high\ - \ school chemistry.\n\n" -"group": "mmlu_alt_ov_05a_stem_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_high_school_chemistry_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_computer_science.yaml deleted file mode 100644 index 91c2c896..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_computer_science" -"description": "The following are multiple choice questions (with answers) about high\ - \ school computer science.\n\n" -"group": "mmlu_alt_ov_05a_stem_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_high_school_computer_science_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_european_history.yaml deleted file mode 100644 index 4808b8a9..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_european_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_european_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school european history.\n\n" -"group": "mmlu_alt_ov_05a_humanities_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_high_school_european_history_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_geography.yaml deleted file mode 100644 index cc97699f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_geography.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_geography" -"description": "The following are multiple choice questions (with answers) about high\ - \ school geography.\n\n" -"group": "mmlu_alt_ov_05a_social_sciences_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_high_school_geography_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_government_and_politics.yaml deleted file mode 100644 index e47ee7fe..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_government_and_politics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_government_and_politics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school government and politics.\n\n" -"group": "mmlu_alt_ov_05a_social_sciences_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_high_school_government_and_politics_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_macroeconomics.yaml deleted file mode 100644 index d2482b5d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_macroeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_macroeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school macroeconomics.\n\n" -"group": "mmlu_alt_ov_05a_social_sciences_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_high_school_macroeconomics_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_mathematics.yaml deleted file mode 100644 index 4ebef56b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_mathematics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school mathematics.\n\n" -"group": "mmlu_alt_ov_05a_stem_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_high_school_mathematics_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_microeconomics.yaml deleted file mode 100644 index 3c13d5ae..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_microeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_microeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school microeconomics.\n\n" -"group": "mmlu_alt_ov_05a_social_sciences_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_high_school_microeconomics_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_physics.yaml deleted file mode 100644 index 692b26e6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_physics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school physics.\n\n" -"group": "mmlu_alt_ov_05a_stem_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_high_school_physics_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_psychology.yaml deleted file mode 100644 index 581d7b06..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_psychology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school psychology.\n\n" -"group": "mmlu_alt_ov_05a_social_sciences_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_high_school_psychology_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_statistics.yaml deleted file mode 100644 index a23f4d65..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_statistics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_statistics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school statistics.\n\n" -"group": "mmlu_alt_ov_05a_stem_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_high_school_statistics_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_us_history.yaml deleted file mode 100644 index 4d859b9b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_us_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_us_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school us history.\n\n" -"group": "mmlu_alt_ov_05a_humanities_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_high_school_us_history_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_world_history.yaml deleted file mode 100644 index b319fea5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_high_school_world_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_world_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school world history.\n\n" -"group": "mmlu_alt_ov_05a_humanities_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_high_school_world_history_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_human_aging.yaml deleted file mode 100644 index 4a45794f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_human_aging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_aging" -"description": "The following are multiple choice questions (with answers) about human\ - \ aging.\n\n" -"group": "mmlu_alt_ov_05a_other_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_human_aging_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_human_sexuality.yaml deleted file mode 100644 index 594adb32..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_human_sexuality.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_sexuality" -"description": "The following are multiple choice questions (with answers) about human\ - \ sexuality.\n\n" -"group": "mmlu_alt_ov_05a_social_sciences_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_human_sexuality_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_international_law.yaml deleted file mode 100644 index 37de1285..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_international_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "international_law" -"description": "The following are multiple choice questions (with answers) about international\ - \ law.\n\n" -"group": "mmlu_alt_ov_05a_humanities_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_international_law_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_jurisprudence.yaml deleted file mode 100644 index a131b0a5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_jurisprudence.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "jurisprudence" -"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ - \n" -"group": "mmlu_alt_ov_05a_humanities_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_jurisprudence_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_logical_fallacies.yaml deleted file mode 100644 index 017ee095..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_logical_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "logical_fallacies" -"description": "The following are multiple choice questions (with answers) about logical\ - \ fallacies.\n\n" -"group": "mmlu_alt_ov_05a_humanities_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_logical_fallacies_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_machine_learning.yaml deleted file mode 100644 index eb47bedb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_machine_learning.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "machine_learning" -"description": "The following are multiple choice questions (with answers) about machine\ - \ learning.\n\n" -"group": "mmlu_alt_ov_05a_stem_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_machine_learning_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_management.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_management.yaml deleted file mode 100644 index dfc6cc7b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_management.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "management" -"description": "The following are multiple choice questions (with answers) about management.\n\ - \n" -"group": "mmlu_alt_ov_05a_other_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_management_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_marketing.yaml deleted file mode 100644 index 43764549..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_marketing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "marketing" -"description": "The following are multiple choice questions (with answers) about marketing.\n\ - \n" -"group": "mmlu_alt_ov_05a_other_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_marketing_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_medical_genetics.yaml deleted file mode 100644 index dd1e31fc..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_medical_genetics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "medical_genetics" -"description": "The following are multiple choice questions (with answers) about medical\ - \ genetics.\n\n" -"group": "mmlu_alt_ov_05a_other_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_medical_genetics_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_miscellaneous.yaml deleted file mode 100644 index 1e0c4d1e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_miscellaneous.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "miscellaneous" -"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ - \n" -"group": "mmlu_alt_ov_05a_other_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_miscellaneous_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_moral_disputes.yaml deleted file mode 100644 index 52ce7c15..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_moral_disputes.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_disputes" -"description": "The following are multiple choice questions (with answers) about moral\ - \ disputes.\n\n" -"group": "mmlu_alt_ov_05a_humanities_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_moral_disputes_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_moral_scenarios.yaml deleted file mode 100644 index cfd17a03..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_moral_scenarios.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_scenarios" -"description": "The following are multiple choice questions (with answers) about moral\ - \ scenarios.\n\n" -"group": "mmlu_alt_ov_05a_humanities_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_moral_scenarios_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_nutrition.yaml deleted file mode 100644 index f8325f52..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_nutrition.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "nutrition" -"description": "The following are multiple choice questions (with answers) about nutrition.\n\ - \n" -"group": "mmlu_alt_ov_05a_other_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_nutrition_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_philosophy.yaml deleted file mode 100644 index aa7695c2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_philosophy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "philosophy" -"description": "The following are multiple choice questions (with answers) about philosophy.\n\ - \n" -"group": "mmlu_alt_ov_05a_humanities_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_philosophy_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_prehistory.yaml deleted file mode 100644 index 4fa9115f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_prehistory.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "prehistory" -"description": "The following are multiple choice questions (with answers) about prehistory.\n\ - \n" -"group": "mmlu_alt_ov_05a_humanities_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_prehistory_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_professional_accounting.yaml deleted file mode 100644 index 22fcb5a6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_professional_accounting.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_accounting" -"description": "The following are multiple choice questions (with answers) about professional\ - \ accounting.\n\n" -"group": "mmlu_alt_ov_05a_other_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_professional_accounting_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_professional_law.yaml deleted file mode 100644 index f2c7af01..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_professional_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_law" -"description": "The following are multiple choice questions (with answers) about professional\ - \ law.\n\n" -"group": "mmlu_alt_ov_05a_humanities_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_professional_law_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_professional_medicine.yaml deleted file mode 100644 index 27f6ba3f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_professional_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_medicine" -"description": "The following are multiple choice questions (with answers) about professional\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_05a_other_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_professional_medicine_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_professional_psychology.yaml deleted file mode 100644 index 18e49d01..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_professional_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_psychology" -"description": "The following are multiple choice questions (with answers) about professional\ - \ psychology.\n\n" -"group": "mmlu_alt_ov_05a_social_sciences_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_professional_psychology_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_public_relations.yaml deleted file mode 100644 index 387e76b6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_public_relations.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "public_relations" -"description": "The following are multiple choice questions (with answers) about public\ - \ relations.\n\n" -"group": "mmlu_alt_ov_05a_social_sciences_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_public_relations_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_security_studies.yaml deleted file mode 100644 index 333651c5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_security_studies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "security_studies" -"description": "The following are multiple choice questions (with answers) about security\ - \ studies.\n\n" -"group": "mmlu_alt_ov_05a_social_sciences_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_security_studies_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_sociology.yaml deleted file mode 100644 index 27612794..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_sociology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sociology" -"description": "The following are multiple choice questions (with answers) about sociology.\n\ - \n" -"group": "mmlu_alt_ov_05a_social_sciences_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_sociology_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_us_foreign_policy.yaml deleted file mode 100644 index 522b3012..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_us_foreign_policy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "us_foreign_policy" -"description": "The following are multiple choice questions (with answers) about us\ - \ foreign policy.\n\n" -"group": "mmlu_alt_ov_05a_social_sciences_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_us_foreign_policy_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_virology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_virology.yaml deleted file mode 100644 index eaeed5d0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_virology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "virology" -"description": "The following are multiple choice questions (with answers) about virology.\n\ - \n" -"group": "mmlu_alt_ov_05a_other_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_virology_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_world_religions.yaml deleted file mode 100644 index 8eb12645..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_world_religions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "world_religions" -"description": "The following are multiple choice questions (with answers) about world\ - \ religions.\n\n" -"group": "mmlu_alt_ov_05a_humanities_generative" -"include": "_template_yaml" -"task": "mmlu_alt_ov_05a_world_religions_generative" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/_mmlu_ov_06.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/_mmlu_ov_06.yaml deleted file mode 100644 index c5e53e73..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/_mmlu_ov_06.yaml +++ /dev/null @@ -1,5 +0,0 @@ -group: mmlu_alt_ov_06 -task: - - mmlu_alt_ov_06a - - mmlu_alt_ov_06b - - mmlu_alt_ov_06c diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/_mmlu.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/_mmlu.yaml deleted file mode 100644 index 718f8375..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/_mmlu.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: mmlu_alt_ov_06a -task: - - mmlu_alt_ov_06a_stem - - mmlu_alt_ov_06a_other - - mmlu_alt_ov_06a_social_sciences - - mmlu_alt_ov_06a_humanities diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/_template_yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/_template_yaml deleted file mode 100644 index 5269f145..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/_template_yaml +++ /dev/null @@ -1,11 +0,0 @@ -dataset_path: hails/mmlu_no_train -test_split: test -fewshot_split: dev -output_type: multiple_choice -doc_to_text: !function ../../../styles.template_06 -doc_to_choice: !function ../../../styles.choice_06a -doc_to_target: answer -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_abstract_algebra.yaml deleted file mode 100644 index 4e50dad1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_abstract_algebra.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "abstract_algebra" -"description": "The following are multiple choice questions (with answers) about abstract\ - \ algebra.\n\n" -"group": "mmlu_alt_ov_06a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_anatomy.yaml deleted file mode 100644 index 30b360b6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_anatomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "anatomy" -"description": "The following are multiple choice questions (with answers) about anatomy.\n\ - \n" -"group": "mmlu_alt_ov_06a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_anatomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_astronomy.yaml deleted file mode 100644 index a766a6e4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_astronomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "astronomy" -"description": "The following are multiple choice questions (with answers) about astronomy.\n\ - \n" -"group": "mmlu_alt_ov_06a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_astronomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_business_ethics.yaml deleted file mode 100644 index 6f8f47b4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_business_ethics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "business_ethics" -"description": "The following are multiple choice questions (with answers) about business\ - \ ethics.\n\n" -"group": "mmlu_alt_ov_06a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_business_ethics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_clinical_knowledge.yaml deleted file mode 100644 index 1900d331..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_clinical_knowledge.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "clinical_knowledge" -"description": "The following are multiple choice questions (with answers) about clinical\ - \ knowledge.\n\n" -"group": "mmlu_alt_ov_06a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_college_biology.yaml deleted file mode 100644 index de67fe09..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_college_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_biology" -"description": "The following are multiple choice questions (with answers) about college\ - \ biology.\n\n" -"group": "mmlu_alt_ov_06a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_college_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_college_chemistry.yaml deleted file mode 100644 index 9bc65269..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_college_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_chemistry" -"description": "The following are multiple choice questions (with answers) about college\ - \ chemistry.\n\n" -"group": "mmlu_alt_ov_06a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_college_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_college_computer_science.yaml deleted file mode 100644 index 94ad7a72..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_college_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_computer_science" -"description": "The following are multiple choice questions (with answers) about college\ - \ computer science.\n\n" -"group": "mmlu_alt_ov_06a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_college_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_college_mathematics.yaml deleted file mode 100644 index f536b141..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_college_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_mathematics" -"description": "The following are multiple choice questions (with answers) about college\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_06a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_college_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_college_medicine.yaml deleted file mode 100644 index 5dd00361..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_college_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_medicine" -"description": "The following are multiple choice questions (with answers) about college\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_06a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_college_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_college_physics.yaml deleted file mode 100644 index b27f03ef..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_college_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_physics" -"description": "The following are multiple choice questions (with answers) about college\ - \ physics.\n\n" -"group": "mmlu_alt_ov_06a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_college_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_computer_security.yaml deleted file mode 100644 index 03ac40dc..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_computer_security.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "computer_security" -"description": "The following are multiple choice questions (with answers) about computer\ - \ security.\n\n" -"group": "mmlu_alt_ov_06a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_computer_security" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_conceptual_physics.yaml deleted file mode 100644 index b57076e3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_conceptual_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "conceptual_physics" -"description": "The following are multiple choice questions (with answers) about conceptual\ - \ physics.\n\n" -"group": "mmlu_alt_ov_06a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_econometrics.yaml deleted file mode 100644 index 479795ca..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_econometrics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "econometrics" -"description": "The following are multiple choice questions (with answers) about econometrics.\n\ - \n" -"group": "mmlu_alt_ov_06a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_econometrics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_electrical_engineering.yaml deleted file mode 100644 index 3659eea6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_electrical_engineering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "electrical_engineering" -"description": "The following are multiple choice questions (with answers) about electrical\ - \ engineering.\n\n" -"group": "mmlu_alt_ov_06a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_elementary_mathematics.yaml deleted file mode 100644 index 551cb12e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_elementary_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "elementary_mathematics" -"description": "The following are multiple choice questions (with answers) about elementary\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_06a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_formal_logic.yaml deleted file mode 100644 index d796f826..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_formal_logic.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_logic" -"description": "The following are multiple choice questions (with answers) about formal\ - \ logic.\n\n" -"group": "mmlu_alt_ov_06a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_formal_logic" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_global_facts.yaml deleted file mode 100644 index d86149e3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_global_facts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "global_facts" -"description": "The following are multiple choice questions (with answers) about global\ - \ facts.\n\n" -"group": "mmlu_alt_ov_06a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_global_facts" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_biology.yaml deleted file mode 100644 index 67cc26ac..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_biology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school biology.\n\n" -"group": "mmlu_alt_ov_06a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_high_school_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_chemistry.yaml deleted file mode 100644 index 9a4e2e0c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_chemistry" -"description": "The following are multiple choice questions (with answers) about high\ - \ school chemistry.\n\n" -"group": "mmlu_alt_ov_06a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_computer_science.yaml deleted file mode 100644 index 8735604d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_computer_science" -"description": "The following are multiple choice questions (with answers) about high\ - \ school computer science.\n\n" -"group": "mmlu_alt_ov_06a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_european_history.yaml deleted file mode 100644 index 4d6d7008..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_european_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_european_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school european history.\n\n" -"group": "mmlu_alt_ov_06a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_geography.yaml deleted file mode 100644 index af9df9f6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_geography.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_geography" -"description": "The following are multiple choice questions (with answers) about high\ - \ school geography.\n\n" -"group": "mmlu_alt_ov_06a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_high_school_geography" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_government_and_politics.yaml deleted file mode 100644 index 7ef68e44..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_government_and_politics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_government_and_politics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school government and politics.\n\n" -"group": "mmlu_alt_ov_06a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_macroeconomics.yaml deleted file mode 100644 index f2348869..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_macroeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_macroeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school macroeconomics.\n\n" -"group": "mmlu_alt_ov_06a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_mathematics.yaml deleted file mode 100644 index 1d67a878..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_mathematics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school mathematics.\n\n" -"group": "mmlu_alt_ov_06a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_microeconomics.yaml deleted file mode 100644 index 4d571b4c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_microeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_microeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school microeconomics.\n\n" -"group": "mmlu_alt_ov_06a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_physics.yaml deleted file mode 100644 index 27c50c84..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_physics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school physics.\n\n" -"group": "mmlu_alt_ov_06a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_high_school_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_psychology.yaml deleted file mode 100644 index e7062280..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_psychology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school psychology.\n\n" -"group": "mmlu_alt_ov_06a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_statistics.yaml deleted file mode 100644 index fee1a14b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_statistics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_statistics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school statistics.\n\n" -"group": "mmlu_alt_ov_06a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_us_history.yaml deleted file mode 100644 index 6e77d57d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_us_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_us_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school us history.\n\n" -"group": "mmlu_alt_ov_06a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_world_history.yaml deleted file mode 100644 index 938a13aa..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_high_school_world_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_world_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school world history.\n\n" -"group": "mmlu_alt_ov_06a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_human_aging.yaml deleted file mode 100644 index 1e8b1113..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_human_aging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_aging" -"description": "The following are multiple choice questions (with answers) about human\ - \ aging.\n\n" -"group": "mmlu_alt_ov_06a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_human_aging" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_human_sexuality.yaml deleted file mode 100644 index 80840d8d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_human_sexuality.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_sexuality" -"description": "The following are multiple choice questions (with answers) about human\ - \ sexuality.\n\n" -"group": "mmlu_alt_ov_06a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_human_sexuality" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_international_law.yaml deleted file mode 100644 index a4efe485..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_international_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "international_law" -"description": "The following are multiple choice questions (with answers) about international\ - \ law.\n\n" -"group": "mmlu_alt_ov_06a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_international_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_jurisprudence.yaml deleted file mode 100644 index 31195387..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_jurisprudence.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "jurisprudence" -"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ - \n" -"group": "mmlu_alt_ov_06a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_jurisprudence" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_logical_fallacies.yaml deleted file mode 100644 index 724ac307..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_logical_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "logical_fallacies" -"description": "The following are multiple choice questions (with answers) about logical\ - \ fallacies.\n\n" -"group": "mmlu_alt_ov_06a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_machine_learning.yaml deleted file mode 100644 index dc2b7f06..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_machine_learning.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "machine_learning" -"description": "The following are multiple choice questions (with answers) about machine\ - \ learning.\n\n" -"group": "mmlu_alt_ov_06a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_machine_learning" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_management.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_management.yaml deleted file mode 100644 index c5c1cb69..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_management.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "management" -"description": "The following are multiple choice questions (with answers) about management.\n\ - \n" -"group": "mmlu_alt_ov_06a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_management" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_marketing.yaml deleted file mode 100644 index 635b5dbc..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_marketing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "marketing" -"description": "The following are multiple choice questions (with answers) about marketing.\n\ - \n" -"group": "mmlu_alt_ov_06a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_marketing" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_medical_genetics.yaml deleted file mode 100644 index 0100fec1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_medical_genetics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "medical_genetics" -"description": "The following are multiple choice questions (with answers) about medical\ - \ genetics.\n\n" -"group": "mmlu_alt_ov_06a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_medical_genetics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_miscellaneous.yaml deleted file mode 100644 index 05172964..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_miscellaneous.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "miscellaneous" -"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ - \n" -"group": "mmlu_alt_ov_06a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_miscellaneous" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_moral_disputes.yaml deleted file mode 100644 index 6f53f46b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_moral_disputes.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_disputes" -"description": "The following are multiple choice questions (with answers) about moral\ - \ disputes.\n\n" -"group": "mmlu_alt_ov_06a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_moral_disputes" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_moral_scenarios.yaml deleted file mode 100644 index c551b61a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_moral_scenarios.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_scenarios" -"description": "The following are multiple choice questions (with answers) about moral\ - \ scenarios.\n\n" -"group": "mmlu_alt_ov_06a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_nutrition.yaml deleted file mode 100644 index 63b9fbf5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_nutrition.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "nutrition" -"description": "The following are multiple choice questions (with answers) about nutrition.\n\ - \n" -"group": "mmlu_alt_ov_06a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_nutrition" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_philosophy.yaml deleted file mode 100644 index 942ac9e6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_philosophy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "philosophy" -"description": "The following are multiple choice questions (with answers) about philosophy.\n\ - \n" -"group": "mmlu_alt_ov_06a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_philosophy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_prehistory.yaml deleted file mode 100644 index ca6b4e37..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_prehistory.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "prehistory" -"description": "The following are multiple choice questions (with answers) about prehistory.\n\ - \n" -"group": "mmlu_alt_ov_06a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_prehistory" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_professional_accounting.yaml deleted file mode 100644 index 9fc815a4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_professional_accounting.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_accounting" -"description": "The following are multiple choice questions (with answers) about professional\ - \ accounting.\n\n" -"group": "mmlu_alt_ov_06a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_professional_accounting" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_professional_law.yaml deleted file mode 100644 index ed08dc12..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_professional_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_law" -"description": "The following are multiple choice questions (with answers) about professional\ - \ law.\n\n" -"group": "mmlu_alt_ov_06a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_professional_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_professional_medicine.yaml deleted file mode 100644 index 8542b318..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_professional_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_medicine" -"description": "The following are multiple choice questions (with answers) about professional\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_06a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_professional_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_professional_psychology.yaml deleted file mode 100644 index 85b49a34..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_professional_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_psychology" -"description": "The following are multiple choice questions (with answers) about professional\ - \ psychology.\n\n" -"group": "mmlu_alt_ov_06a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_professional_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_public_relations.yaml deleted file mode 100644 index 1cf7dd2a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_public_relations.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "public_relations" -"description": "The following are multiple choice questions (with answers) about public\ - \ relations.\n\n" -"group": "mmlu_alt_ov_06a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_public_relations" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_security_studies.yaml deleted file mode 100644 index 7fe72f47..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_security_studies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "security_studies" -"description": "The following are multiple choice questions (with answers) about security\ - \ studies.\n\n" -"group": "mmlu_alt_ov_06a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_security_studies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_sociology.yaml deleted file mode 100644 index 0c034af4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_sociology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sociology" -"description": "The following are multiple choice questions (with answers) about sociology.\n\ - \n" -"group": "mmlu_alt_ov_06a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_sociology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_us_foreign_policy.yaml deleted file mode 100644 index 3f915db0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_us_foreign_policy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "us_foreign_policy" -"description": "The following are multiple choice questions (with answers) about us\ - \ foreign policy.\n\n" -"group": "mmlu_alt_ov_06a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_virology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_virology.yaml deleted file mode 100644 index db1a318b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_virology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "virology" -"description": "The following are multiple choice questions (with answers) about virology.\n\ - \n" -"group": "mmlu_alt_ov_06a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_virology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_world_religions.yaml deleted file mode 100644 index b39f57a3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/a/mmlu_world_religions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "world_religions" -"description": "The following are multiple choice questions (with answers) about world\ - \ religions.\n\n" -"group": "mmlu_alt_ov_06a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06a_world_religions" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/_mmlu.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/_mmlu.yaml deleted file mode 100644 index 76001890..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/_mmlu.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: mmlu_alt_ov_06b -task: - - mmlu_alt_ov_06b_stem - - mmlu_alt_ov_06b_other - - mmlu_alt_ov_06b_social_sciences - - mmlu_alt_ov_06b_humanities diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/_template_yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/_template_yaml deleted file mode 100644 index fc649845..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/_template_yaml +++ /dev/null @@ -1,11 +0,0 @@ -dataset_path: hails/mmlu_no_train -test_split: test -fewshot_split: dev -output_type: multiple_choice -doc_to_text: !function ../../../styles.template_06 -doc_to_choice: !function ../../../styles.choice_06b -doc_to_target: answer -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_abstract_algebra.yaml deleted file mode 100644 index 6179fecd..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_abstract_algebra.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "abstract_algebra" -"description": "The following are multiple choice questions (with answers) about abstract\ - \ algebra.\n\n" -"group": "mmlu_alt_ov_06b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_anatomy.yaml deleted file mode 100644 index 8dc6e13e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_anatomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "anatomy" -"description": "The following are multiple choice questions (with answers) about anatomy.\n\ - \n" -"group": "mmlu_alt_ov_06b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_anatomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_astronomy.yaml deleted file mode 100644 index e3cb4008..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_astronomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "astronomy" -"description": "The following are multiple choice questions (with answers) about astronomy.\n\ - \n" -"group": "mmlu_alt_ov_06b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_astronomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_business_ethics.yaml deleted file mode 100644 index 73bca569..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_business_ethics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "business_ethics" -"description": "The following are multiple choice questions (with answers) about business\ - \ ethics.\n\n" -"group": "mmlu_alt_ov_06b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_business_ethics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_clinical_knowledge.yaml deleted file mode 100644 index 004d1f96..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_clinical_knowledge.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "clinical_knowledge" -"description": "The following are multiple choice questions (with answers) about clinical\ - \ knowledge.\n\n" -"group": "mmlu_alt_ov_06b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_college_biology.yaml deleted file mode 100644 index f1f1b9d6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_college_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_biology" -"description": "The following are multiple choice questions (with answers) about college\ - \ biology.\n\n" -"group": "mmlu_alt_ov_06b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_college_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_college_chemistry.yaml deleted file mode 100644 index d2cb94a4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_college_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_chemistry" -"description": "The following are multiple choice questions (with answers) about college\ - \ chemistry.\n\n" -"group": "mmlu_alt_ov_06b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_college_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_college_computer_science.yaml deleted file mode 100644 index dafba5c3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_college_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_computer_science" -"description": "The following are multiple choice questions (with answers) about college\ - \ computer science.\n\n" -"group": "mmlu_alt_ov_06b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_college_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_college_mathematics.yaml deleted file mode 100644 index 17fea247..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_college_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_mathematics" -"description": "The following are multiple choice questions (with answers) about college\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_06b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_college_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_college_medicine.yaml deleted file mode 100644 index b654b53d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_college_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_medicine" -"description": "The following are multiple choice questions (with answers) about college\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_06b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_college_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_college_physics.yaml deleted file mode 100644 index e3e20c54..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_college_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_physics" -"description": "The following are multiple choice questions (with answers) about college\ - \ physics.\n\n" -"group": "mmlu_alt_ov_06b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_college_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_computer_security.yaml deleted file mode 100644 index b59eaf74..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_computer_security.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "computer_security" -"description": "The following are multiple choice questions (with answers) about computer\ - \ security.\n\n" -"group": "mmlu_alt_ov_06b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_computer_security" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_conceptual_physics.yaml deleted file mode 100644 index 17a7f402..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_conceptual_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "conceptual_physics" -"description": "The following are multiple choice questions (with answers) about conceptual\ - \ physics.\n\n" -"group": "mmlu_alt_ov_06b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_econometrics.yaml deleted file mode 100644 index 67fd25f1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_econometrics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "econometrics" -"description": "The following are multiple choice questions (with answers) about econometrics.\n\ - \n" -"group": "mmlu_alt_ov_06b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_econometrics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_electrical_engineering.yaml deleted file mode 100644 index 09682ca0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_electrical_engineering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "electrical_engineering" -"description": "The following are multiple choice questions (with answers) about electrical\ - \ engineering.\n\n" -"group": "mmlu_alt_ov_06b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_elementary_mathematics.yaml deleted file mode 100644 index 332a7695..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_elementary_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "elementary_mathematics" -"description": "The following are multiple choice questions (with answers) about elementary\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_06b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_formal_logic.yaml deleted file mode 100644 index 9812ff4e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_formal_logic.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_logic" -"description": "The following are multiple choice questions (with answers) about formal\ - \ logic.\n\n" -"group": "mmlu_alt_ov_06b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_formal_logic" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_global_facts.yaml deleted file mode 100644 index af72cb2c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_global_facts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "global_facts" -"description": "The following are multiple choice questions (with answers) about global\ - \ facts.\n\n" -"group": "mmlu_alt_ov_06b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_global_facts" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_biology.yaml deleted file mode 100644 index 9ce58fa3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_biology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school biology.\n\n" -"group": "mmlu_alt_ov_06b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_high_school_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_chemistry.yaml deleted file mode 100644 index 15f2a06c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_chemistry" -"description": "The following are multiple choice questions (with answers) about high\ - \ school chemistry.\n\n" -"group": "mmlu_alt_ov_06b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_computer_science.yaml deleted file mode 100644 index e2d857d4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_computer_science" -"description": "The following are multiple choice questions (with answers) about high\ - \ school computer science.\n\n" -"group": "mmlu_alt_ov_06b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_european_history.yaml deleted file mode 100644 index 51491ad2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_european_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_european_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school european history.\n\n" -"group": "mmlu_alt_ov_06b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_geography.yaml deleted file mode 100644 index b3bdd3a1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_geography.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_geography" -"description": "The following are multiple choice questions (with answers) about high\ - \ school geography.\n\n" -"group": "mmlu_alt_ov_06b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_high_school_geography" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_government_and_politics.yaml deleted file mode 100644 index 84250124..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_government_and_politics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_government_and_politics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school government and politics.\n\n" -"group": "mmlu_alt_ov_06b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_macroeconomics.yaml deleted file mode 100644 index 7b149fd5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_macroeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_macroeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school macroeconomics.\n\n" -"group": "mmlu_alt_ov_06b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_mathematics.yaml deleted file mode 100644 index 73ded429..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_mathematics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school mathematics.\n\n" -"group": "mmlu_alt_ov_06b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_microeconomics.yaml deleted file mode 100644 index e26eb966..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_microeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_microeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school microeconomics.\n\n" -"group": "mmlu_alt_ov_06b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_physics.yaml deleted file mode 100644 index 12f27df9..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_physics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school physics.\n\n" -"group": "mmlu_alt_ov_06b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_high_school_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_psychology.yaml deleted file mode 100644 index 9f8c4f3c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_psychology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school psychology.\n\n" -"group": "mmlu_alt_ov_06b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_statistics.yaml deleted file mode 100644 index 4b3547ff..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_statistics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_statistics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school statistics.\n\n" -"group": "mmlu_alt_ov_06b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_us_history.yaml deleted file mode 100644 index 613b44a3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_us_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_us_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school us history.\n\n" -"group": "mmlu_alt_ov_06b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_world_history.yaml deleted file mode 100644 index ed0897c1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_high_school_world_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_world_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school world history.\n\n" -"group": "mmlu_alt_ov_06b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_human_aging.yaml deleted file mode 100644 index dc4a6b07..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_human_aging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_aging" -"description": "The following are multiple choice questions (with answers) about human\ - \ aging.\n\n" -"group": "mmlu_alt_ov_06b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_human_aging" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_human_sexuality.yaml deleted file mode 100644 index d8187a12..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_human_sexuality.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_sexuality" -"description": "The following are multiple choice questions (with answers) about human\ - \ sexuality.\n\n" -"group": "mmlu_alt_ov_06b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_human_sexuality" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_international_law.yaml deleted file mode 100644 index 933f50b8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_international_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "international_law" -"description": "The following are multiple choice questions (with answers) about international\ - \ law.\n\n" -"group": "mmlu_alt_ov_06b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_international_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_jurisprudence.yaml deleted file mode 100644 index ae748888..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_jurisprudence.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "jurisprudence" -"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ - \n" -"group": "mmlu_alt_ov_06b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_jurisprudence" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_logical_fallacies.yaml deleted file mode 100644 index 51de0bab..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_logical_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "logical_fallacies" -"description": "The following are multiple choice questions (with answers) about logical\ - \ fallacies.\n\n" -"group": "mmlu_alt_ov_06b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_machine_learning.yaml deleted file mode 100644 index cbc36e6d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_machine_learning.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "machine_learning" -"description": "The following are multiple choice questions (with answers) about machine\ - \ learning.\n\n" -"group": "mmlu_alt_ov_06b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_machine_learning" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_management.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_management.yaml deleted file mode 100644 index 0faec6dd..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_management.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "management" -"description": "The following are multiple choice questions (with answers) about management.\n\ - \n" -"group": "mmlu_alt_ov_06b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_management" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_marketing.yaml deleted file mode 100644 index c690b7c0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_marketing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "marketing" -"description": "The following are multiple choice questions (with answers) about marketing.\n\ - \n" -"group": "mmlu_alt_ov_06b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_marketing" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_medical_genetics.yaml deleted file mode 100644 index 169412c6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_medical_genetics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "medical_genetics" -"description": "The following are multiple choice questions (with answers) about medical\ - \ genetics.\n\n" -"group": "mmlu_alt_ov_06b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_medical_genetics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_miscellaneous.yaml deleted file mode 100644 index 0fda3a54..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_miscellaneous.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "miscellaneous" -"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ - \n" -"group": "mmlu_alt_ov_06b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_miscellaneous" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_moral_disputes.yaml deleted file mode 100644 index 5289c359..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_moral_disputes.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_disputes" -"description": "The following are multiple choice questions (with answers) about moral\ - \ disputes.\n\n" -"group": "mmlu_alt_ov_06b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_moral_disputes" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_moral_scenarios.yaml deleted file mode 100644 index db749c8c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_moral_scenarios.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_scenarios" -"description": "The following are multiple choice questions (with answers) about moral\ - \ scenarios.\n\n" -"group": "mmlu_alt_ov_06b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_nutrition.yaml deleted file mode 100644 index c33d1bdc..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_nutrition.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "nutrition" -"description": "The following are multiple choice questions (with answers) about nutrition.\n\ - \n" -"group": "mmlu_alt_ov_06b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_nutrition" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_philosophy.yaml deleted file mode 100644 index 247eab86..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_philosophy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "philosophy" -"description": "The following are multiple choice questions (with answers) about philosophy.\n\ - \n" -"group": "mmlu_alt_ov_06b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_philosophy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_prehistory.yaml deleted file mode 100644 index eb87bb71..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_prehistory.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "prehistory" -"description": "The following are multiple choice questions (with answers) about prehistory.\n\ - \n" -"group": "mmlu_alt_ov_06b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_prehistory" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_professional_accounting.yaml deleted file mode 100644 index e65e27de..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_professional_accounting.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_accounting" -"description": "The following are multiple choice questions (with answers) about professional\ - \ accounting.\n\n" -"group": "mmlu_alt_ov_06b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_professional_accounting" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_professional_law.yaml deleted file mode 100644 index f8566534..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_professional_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_law" -"description": "The following are multiple choice questions (with answers) about professional\ - \ law.\n\n" -"group": "mmlu_alt_ov_06b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_professional_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_professional_medicine.yaml deleted file mode 100644 index c9ab6259..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_professional_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_medicine" -"description": "The following are multiple choice questions (with answers) about professional\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_06b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_professional_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_professional_psychology.yaml deleted file mode 100644 index 08dccb1c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_professional_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_psychology" -"description": "The following are multiple choice questions (with answers) about professional\ - \ psychology.\n\n" -"group": "mmlu_alt_ov_06b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_professional_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_public_relations.yaml deleted file mode 100644 index 6ac634d6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_public_relations.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "public_relations" -"description": "The following are multiple choice questions (with answers) about public\ - \ relations.\n\n" -"group": "mmlu_alt_ov_06b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_public_relations" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_security_studies.yaml deleted file mode 100644 index 488ebfcf..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_security_studies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "security_studies" -"description": "The following are multiple choice questions (with answers) about security\ - \ studies.\n\n" -"group": "mmlu_alt_ov_06b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_security_studies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_sociology.yaml deleted file mode 100644 index f36a6135..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_sociology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sociology" -"description": "The following are multiple choice questions (with answers) about sociology.\n\ - \n" -"group": "mmlu_alt_ov_06b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_sociology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_us_foreign_policy.yaml deleted file mode 100644 index 4c4a543a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_us_foreign_policy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "us_foreign_policy" -"description": "The following are multiple choice questions (with answers) about us\ - \ foreign policy.\n\n" -"group": "mmlu_alt_ov_06b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_virology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_virology.yaml deleted file mode 100644 index a11b29e3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_virology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "virology" -"description": "The following are multiple choice questions (with answers) about virology.\n\ - \n" -"group": "mmlu_alt_ov_06b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_virology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_world_religions.yaml deleted file mode 100644 index 497b55b1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/b/mmlu_world_religions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "world_religions" -"description": "The following are multiple choice questions (with answers) about world\ - \ religions.\n\n" -"group": "mmlu_alt_ov_06b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06b_world_religions" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/_mmlu.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/_mmlu.yaml deleted file mode 100644 index 1397600a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/_mmlu.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: mmlu_alt_ov_06c -task: - - mmlu_alt_ov_06c_stem - - mmlu_alt_ov_06c_other - - mmlu_alt_ov_06c_social_sciences - - mmlu_alt_ov_06c_humanities diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/_template_yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/_template_yaml deleted file mode 100644 index 414b7b25..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/_template_yaml +++ /dev/null @@ -1,11 +0,0 @@ -dataset_path: hails/mmlu_no_train -test_split: test -fewshot_split: dev -output_type: multiple_choice -doc_to_text: !function ../../../styles.template_06 -doc_to_choice: !function ../../../styles.choice_06c -doc_to_target: answer -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_abstract_algebra.yaml deleted file mode 100644 index 69c61fe4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_abstract_algebra.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "abstract_algebra" -"description": "The following are multiple choice questions (with answers) about abstract\ - \ algebra.\n\n" -"group": "mmlu_alt_ov_06c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_anatomy.yaml deleted file mode 100644 index 42b30178..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_anatomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "anatomy" -"description": "The following are multiple choice questions (with answers) about anatomy.\n\ - \n" -"group": "mmlu_alt_ov_06c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_anatomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_astronomy.yaml deleted file mode 100644 index 7e984f02..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_astronomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "astronomy" -"description": "The following are multiple choice questions (with answers) about astronomy.\n\ - \n" -"group": "mmlu_alt_ov_06c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_astronomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_business_ethics.yaml deleted file mode 100644 index a0ba2227..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_business_ethics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "business_ethics" -"description": "The following are multiple choice questions (with answers) about business\ - \ ethics.\n\n" -"group": "mmlu_alt_ov_06c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_business_ethics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_clinical_knowledge.yaml deleted file mode 100644 index 21789723..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_clinical_knowledge.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "clinical_knowledge" -"description": "The following are multiple choice questions (with answers) about clinical\ - \ knowledge.\n\n" -"group": "mmlu_alt_ov_06c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_college_biology.yaml deleted file mode 100644 index c57937fd..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_college_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_biology" -"description": "The following are multiple choice questions (with answers) about college\ - \ biology.\n\n" -"group": "mmlu_alt_ov_06c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_college_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_college_chemistry.yaml deleted file mode 100644 index ec529c9b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_college_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_chemistry" -"description": "The following are multiple choice questions (with answers) about college\ - \ chemistry.\n\n" -"group": "mmlu_alt_ov_06c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_college_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_college_computer_science.yaml deleted file mode 100644 index 88c3565c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_college_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_computer_science" -"description": "The following are multiple choice questions (with answers) about college\ - \ computer science.\n\n" -"group": "mmlu_alt_ov_06c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_college_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_college_mathematics.yaml deleted file mode 100644 index 56395088..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_college_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_mathematics" -"description": "The following are multiple choice questions (with answers) about college\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_06c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_college_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_college_medicine.yaml deleted file mode 100644 index c1584be7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_college_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_medicine" -"description": "The following are multiple choice questions (with answers) about college\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_06c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_college_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_college_physics.yaml deleted file mode 100644 index 07e4753c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_college_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_physics" -"description": "The following are multiple choice questions (with answers) about college\ - \ physics.\n\n" -"group": "mmlu_alt_ov_06c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_college_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_computer_security.yaml deleted file mode 100644 index 2f48cd9c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_computer_security.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "computer_security" -"description": "The following are multiple choice questions (with answers) about computer\ - \ security.\n\n" -"group": "mmlu_alt_ov_06c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_computer_security" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_conceptual_physics.yaml deleted file mode 100644 index f66d241b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_conceptual_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "conceptual_physics" -"description": "The following are multiple choice questions (with answers) about conceptual\ - \ physics.\n\n" -"group": "mmlu_alt_ov_06c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_econometrics.yaml deleted file mode 100644 index 38b1149f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_econometrics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "econometrics" -"description": "The following are multiple choice questions (with answers) about econometrics.\n\ - \n" -"group": "mmlu_alt_ov_06c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_econometrics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_electrical_engineering.yaml deleted file mode 100644 index 4a486072..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_electrical_engineering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "electrical_engineering" -"description": "The following are multiple choice questions (with answers) about electrical\ - \ engineering.\n\n" -"group": "mmlu_alt_ov_06c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_elementary_mathematics.yaml deleted file mode 100644 index f2db4ce6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_elementary_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "elementary_mathematics" -"description": "The following are multiple choice questions (with answers) about elementary\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_06c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_formal_logic.yaml deleted file mode 100644 index 9ebec057..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_formal_logic.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_logic" -"description": "The following are multiple choice questions (with answers) about formal\ - \ logic.\n\n" -"group": "mmlu_alt_ov_06c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_formal_logic" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_global_facts.yaml deleted file mode 100644 index 9676d885..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_global_facts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "global_facts" -"description": "The following are multiple choice questions (with answers) about global\ - \ facts.\n\n" -"group": "mmlu_alt_ov_06c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_global_facts" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_biology.yaml deleted file mode 100644 index 30fe645b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_biology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school biology.\n\n" -"group": "mmlu_alt_ov_06c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_high_school_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_chemistry.yaml deleted file mode 100644 index 86251d8b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_chemistry" -"description": "The following are multiple choice questions (with answers) about high\ - \ school chemistry.\n\n" -"group": "mmlu_alt_ov_06c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_computer_science.yaml deleted file mode 100644 index ab05057b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_computer_science" -"description": "The following are multiple choice questions (with answers) about high\ - \ school computer science.\n\n" -"group": "mmlu_alt_ov_06c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_european_history.yaml deleted file mode 100644 index f2c07f9b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_european_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_european_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school european history.\n\n" -"group": "mmlu_alt_ov_06c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_geography.yaml deleted file mode 100644 index 0bc8c6a7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_geography.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_geography" -"description": "The following are multiple choice questions (with answers) about high\ - \ school geography.\n\n" -"group": "mmlu_alt_ov_06c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_high_school_geography" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_government_and_politics.yaml deleted file mode 100644 index 47308b25..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_government_and_politics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_government_and_politics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school government and politics.\n\n" -"group": "mmlu_alt_ov_06c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_macroeconomics.yaml deleted file mode 100644 index b1687478..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_macroeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_macroeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school macroeconomics.\n\n" -"group": "mmlu_alt_ov_06c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_mathematics.yaml deleted file mode 100644 index b950a7a0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_mathematics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school mathematics.\n\n" -"group": "mmlu_alt_ov_06c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_microeconomics.yaml deleted file mode 100644 index 38eeef56..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_microeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_microeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school microeconomics.\n\n" -"group": "mmlu_alt_ov_06c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_physics.yaml deleted file mode 100644 index 15da1569..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_physics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school physics.\n\n" -"group": "mmlu_alt_ov_06c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_high_school_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_psychology.yaml deleted file mode 100644 index 975d34ac..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_psychology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school psychology.\n\n" -"group": "mmlu_alt_ov_06c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_statistics.yaml deleted file mode 100644 index 211ff0d8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_statistics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_statistics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school statistics.\n\n" -"group": "mmlu_alt_ov_06c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_us_history.yaml deleted file mode 100644 index 7ddb5bcf..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_us_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_us_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school us history.\n\n" -"group": "mmlu_alt_ov_06c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_world_history.yaml deleted file mode 100644 index 8900bdec..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_high_school_world_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_world_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school world history.\n\n" -"group": "mmlu_alt_ov_06c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_human_aging.yaml deleted file mode 100644 index f8c03c6a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_human_aging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_aging" -"description": "The following are multiple choice questions (with answers) about human\ - \ aging.\n\n" -"group": "mmlu_alt_ov_06c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_human_aging" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_human_sexuality.yaml deleted file mode 100644 index 809864af..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_human_sexuality.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_sexuality" -"description": "The following are multiple choice questions (with answers) about human\ - \ sexuality.\n\n" -"group": "mmlu_alt_ov_06c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_human_sexuality" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_international_law.yaml deleted file mode 100644 index 5fdd16f9..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_international_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "international_law" -"description": "The following are multiple choice questions (with answers) about international\ - \ law.\n\n" -"group": "mmlu_alt_ov_06c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_international_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_jurisprudence.yaml deleted file mode 100644 index 70ad5e92..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_jurisprudence.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "jurisprudence" -"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ - \n" -"group": "mmlu_alt_ov_06c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_jurisprudence" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_logical_fallacies.yaml deleted file mode 100644 index 65822b08..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_logical_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "logical_fallacies" -"description": "The following are multiple choice questions (with answers) about logical\ - \ fallacies.\n\n" -"group": "mmlu_alt_ov_06c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_machine_learning.yaml deleted file mode 100644 index 84ca2350..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_machine_learning.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "machine_learning" -"description": "The following are multiple choice questions (with answers) about machine\ - \ learning.\n\n" -"group": "mmlu_alt_ov_06c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_machine_learning" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_management.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_management.yaml deleted file mode 100644 index 4f442a5a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_management.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "management" -"description": "The following are multiple choice questions (with answers) about management.\n\ - \n" -"group": "mmlu_alt_ov_06c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_management" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_marketing.yaml deleted file mode 100644 index 9fd75423..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_marketing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "marketing" -"description": "The following are multiple choice questions (with answers) about marketing.\n\ - \n" -"group": "mmlu_alt_ov_06c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_marketing" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_medical_genetics.yaml deleted file mode 100644 index 2b1ae607..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_medical_genetics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "medical_genetics" -"description": "The following are multiple choice questions (with answers) about medical\ - \ genetics.\n\n" -"group": "mmlu_alt_ov_06c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_medical_genetics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_miscellaneous.yaml deleted file mode 100644 index dc5ea5d6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_miscellaneous.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "miscellaneous" -"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ - \n" -"group": "mmlu_alt_ov_06c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_miscellaneous" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_moral_disputes.yaml deleted file mode 100644 index 24e16395..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_moral_disputes.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_disputes" -"description": "The following are multiple choice questions (with answers) about moral\ - \ disputes.\n\n" -"group": "mmlu_alt_ov_06c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_moral_disputes" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_moral_scenarios.yaml deleted file mode 100644 index c267865e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_moral_scenarios.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_scenarios" -"description": "The following are multiple choice questions (with answers) about moral\ - \ scenarios.\n\n" -"group": "mmlu_alt_ov_06c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_nutrition.yaml deleted file mode 100644 index ee5bc7da..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_nutrition.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "nutrition" -"description": "The following are multiple choice questions (with answers) about nutrition.\n\ - \n" -"group": "mmlu_alt_ov_06c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_nutrition" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_philosophy.yaml deleted file mode 100644 index ac32db39..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_philosophy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "philosophy" -"description": "The following are multiple choice questions (with answers) about philosophy.\n\ - \n" -"group": "mmlu_alt_ov_06c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_philosophy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_prehistory.yaml deleted file mode 100644 index d8ed3548..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_prehistory.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "prehistory" -"description": "The following are multiple choice questions (with answers) about prehistory.\n\ - \n" -"group": "mmlu_alt_ov_06c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_prehistory" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_professional_accounting.yaml deleted file mode 100644 index ce412083..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_professional_accounting.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_accounting" -"description": "The following are multiple choice questions (with answers) about professional\ - \ accounting.\n\n" -"group": "mmlu_alt_ov_06c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_professional_accounting" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_professional_law.yaml deleted file mode 100644 index ca648a03..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_professional_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_law" -"description": "The following are multiple choice questions (with answers) about professional\ - \ law.\n\n" -"group": "mmlu_alt_ov_06c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_professional_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_professional_medicine.yaml deleted file mode 100644 index 57a16e01..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_professional_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_medicine" -"description": "The following are multiple choice questions (with answers) about professional\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_06c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_professional_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_professional_psychology.yaml deleted file mode 100644 index 728c3325..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_professional_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_psychology" -"description": "The following are multiple choice questions (with answers) about professional\ - \ psychology.\n\n" -"group": "mmlu_alt_ov_06c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_professional_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_public_relations.yaml deleted file mode 100644 index 3a0c30d5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_public_relations.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "public_relations" -"description": "The following are multiple choice questions (with answers) about public\ - \ relations.\n\n" -"group": "mmlu_alt_ov_06c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_public_relations" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_security_studies.yaml deleted file mode 100644 index f396a27f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_security_studies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "security_studies" -"description": "The following are multiple choice questions (with answers) about security\ - \ studies.\n\n" -"group": "mmlu_alt_ov_06c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_security_studies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_sociology.yaml deleted file mode 100644 index ab945977..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_sociology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sociology" -"description": "The following are multiple choice questions (with answers) about sociology.\n\ - \n" -"group": "mmlu_alt_ov_06c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_sociology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_us_foreign_policy.yaml deleted file mode 100644 index 118e3334..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_us_foreign_policy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "us_foreign_policy" -"description": "The following are multiple choice questions (with answers) about us\ - \ foreign policy.\n\n" -"group": "mmlu_alt_ov_06c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_virology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_virology.yaml deleted file mode 100644 index ee29cc4f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_virology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "virology" -"description": "The following are multiple choice questions (with answers) about virology.\n\ - \n" -"group": "mmlu_alt_ov_06c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_virology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_world_religions.yaml deleted file mode 100644 index c48c15b6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_06/c/mmlu_world_religions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "world_religions" -"description": "The following are multiple choice questions (with answers) about world\ - \ religions.\n\n" -"group": "mmlu_alt_ov_06c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_06c_world_religions" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/_mmlu_ov_07.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/_mmlu_ov_07.yaml deleted file mode 100644 index 84452734..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/_mmlu_ov_07.yaml +++ /dev/null @@ -1,5 +0,0 @@ -group: mmlu_alt_ov_07 -task: - - mmlu_alt_ov_07a - - mmlu_alt_ov_07b - - mmlu_alt_ov_07c diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/_mmlu.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/_mmlu.yaml deleted file mode 100644 index e7885ae7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/_mmlu.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: mmlu_alt_ov_07a -task: - - mmlu_alt_ov_07a_stem - - mmlu_alt_ov_07a_other - - mmlu_alt_ov_07a_social_sciences - - mmlu_alt_ov_07a_humanities diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/_template_yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/_template_yaml deleted file mode 100644 index a7932f22..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/_template_yaml +++ /dev/null @@ -1,11 +0,0 @@ -dataset_path: hails/mmlu_no_train -test_split: test -fewshot_split: dev -output_type: multiple_choice -doc_to_text: !function ../../../styles.template_07 -doc_to_choice: !function ../../../styles.choice_07a -doc_to_target: answer -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_abstract_algebra.yaml deleted file mode 100644 index 3753f10f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_abstract_algebra.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "abstract_algebra" -"description": "The following are multiple choice questions (with answers) about abstract\ - \ algebra.\n\n" -"group": "mmlu_alt_ov_07a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_anatomy.yaml deleted file mode 100644 index a8ce274c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_anatomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "anatomy" -"description": "The following are multiple choice questions (with answers) about anatomy.\n\ - \n" -"group": "mmlu_alt_ov_07a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_anatomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_astronomy.yaml deleted file mode 100644 index daa8c358..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_astronomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "astronomy" -"description": "The following are multiple choice questions (with answers) about astronomy.\n\ - \n" -"group": "mmlu_alt_ov_07a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_astronomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_business_ethics.yaml deleted file mode 100644 index 30f52843..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_business_ethics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "business_ethics" -"description": "The following are multiple choice questions (with answers) about business\ - \ ethics.\n\n" -"group": "mmlu_alt_ov_07a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_business_ethics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_clinical_knowledge.yaml deleted file mode 100644 index c5bce599..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_clinical_knowledge.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "clinical_knowledge" -"description": "The following are multiple choice questions (with answers) about clinical\ - \ knowledge.\n\n" -"group": "mmlu_alt_ov_07a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_college_biology.yaml deleted file mode 100644 index fd25e096..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_college_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_biology" -"description": "The following are multiple choice questions (with answers) about college\ - \ biology.\n\n" -"group": "mmlu_alt_ov_07a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_college_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_college_chemistry.yaml deleted file mode 100644 index c7fec3a7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_college_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_chemistry" -"description": "The following are multiple choice questions (with answers) about college\ - \ chemistry.\n\n" -"group": "mmlu_alt_ov_07a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_college_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_college_computer_science.yaml deleted file mode 100644 index e3c5db69..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_college_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_computer_science" -"description": "The following are multiple choice questions (with answers) about college\ - \ computer science.\n\n" -"group": "mmlu_alt_ov_07a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_college_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_college_mathematics.yaml deleted file mode 100644 index 7a30111e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_college_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_mathematics" -"description": "The following are multiple choice questions (with answers) about college\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_07a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_college_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_college_medicine.yaml deleted file mode 100644 index 2148d7f7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_college_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_medicine" -"description": "The following are multiple choice questions (with answers) about college\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_07a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_college_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_college_physics.yaml deleted file mode 100644 index 8c34ca5f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_college_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_physics" -"description": "The following are multiple choice questions (with answers) about college\ - \ physics.\n\n" -"group": "mmlu_alt_ov_07a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_college_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_computer_security.yaml deleted file mode 100644 index 0aeba991..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_computer_security.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "computer_security" -"description": "The following are multiple choice questions (with answers) about computer\ - \ security.\n\n" -"group": "mmlu_alt_ov_07a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_computer_security" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_conceptual_physics.yaml deleted file mode 100644 index bc922c0c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_conceptual_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "conceptual_physics" -"description": "The following are multiple choice questions (with answers) about conceptual\ - \ physics.\n\n" -"group": "mmlu_alt_ov_07a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_econometrics.yaml deleted file mode 100644 index b422078c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_econometrics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "econometrics" -"description": "The following are multiple choice questions (with answers) about econometrics.\n\ - \n" -"group": "mmlu_alt_ov_07a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_econometrics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_electrical_engineering.yaml deleted file mode 100644 index 5dccde22..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_electrical_engineering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "electrical_engineering" -"description": "The following are multiple choice questions (with answers) about electrical\ - \ engineering.\n\n" -"group": "mmlu_alt_ov_07a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_elementary_mathematics.yaml deleted file mode 100644 index b28774fe..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_elementary_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "elementary_mathematics" -"description": "The following are multiple choice questions (with answers) about elementary\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_07a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_formal_logic.yaml deleted file mode 100644 index 7079d212..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_formal_logic.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_logic" -"description": "The following are multiple choice questions (with answers) about formal\ - \ logic.\n\n" -"group": "mmlu_alt_ov_07a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_formal_logic" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_global_facts.yaml deleted file mode 100644 index 20adac4b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_global_facts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "global_facts" -"description": "The following are multiple choice questions (with answers) about global\ - \ facts.\n\n" -"group": "mmlu_alt_ov_07a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_global_facts" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_biology.yaml deleted file mode 100644 index f4ce4a9f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_biology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school biology.\n\n" -"group": "mmlu_alt_ov_07a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_high_school_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_chemistry.yaml deleted file mode 100644 index feeeb165..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_chemistry" -"description": "The following are multiple choice questions (with answers) about high\ - \ school chemistry.\n\n" -"group": "mmlu_alt_ov_07a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_computer_science.yaml deleted file mode 100644 index 4282ecf1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_computer_science" -"description": "The following are multiple choice questions (with answers) about high\ - \ school computer science.\n\n" -"group": "mmlu_alt_ov_07a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_european_history.yaml deleted file mode 100644 index 6a66e634..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_european_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_european_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school european history.\n\n" -"group": "mmlu_alt_ov_07a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_geography.yaml deleted file mode 100644 index 43b64f90..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_geography.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_geography" -"description": "The following are multiple choice questions (with answers) about high\ - \ school geography.\n\n" -"group": "mmlu_alt_ov_07a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_high_school_geography" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_government_and_politics.yaml deleted file mode 100644 index f2c019dc..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_government_and_politics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_government_and_politics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school government and politics.\n\n" -"group": "mmlu_alt_ov_07a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_macroeconomics.yaml deleted file mode 100644 index 89d58a24..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_macroeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_macroeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school macroeconomics.\n\n" -"group": "mmlu_alt_ov_07a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_mathematics.yaml deleted file mode 100644 index 49cb05ae..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_mathematics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school mathematics.\n\n" -"group": "mmlu_alt_ov_07a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_microeconomics.yaml deleted file mode 100644 index bee68ebf..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_microeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_microeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school microeconomics.\n\n" -"group": "mmlu_alt_ov_07a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_physics.yaml deleted file mode 100644 index 70df43b2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_physics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school physics.\n\n" -"group": "mmlu_alt_ov_07a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_high_school_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_psychology.yaml deleted file mode 100644 index 919ea53b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_psychology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school psychology.\n\n" -"group": "mmlu_alt_ov_07a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_statistics.yaml deleted file mode 100644 index a76715c5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_statistics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_statistics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school statistics.\n\n" -"group": "mmlu_alt_ov_07a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_us_history.yaml deleted file mode 100644 index 28f26f92..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_us_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_us_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school us history.\n\n" -"group": "mmlu_alt_ov_07a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_world_history.yaml deleted file mode 100644 index ca94340a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_high_school_world_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_world_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school world history.\n\n" -"group": "mmlu_alt_ov_07a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_human_aging.yaml deleted file mode 100644 index f97857ef..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_human_aging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_aging" -"description": "The following are multiple choice questions (with answers) about human\ - \ aging.\n\n" -"group": "mmlu_alt_ov_07a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_human_aging" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_human_sexuality.yaml deleted file mode 100644 index 389af64c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_human_sexuality.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_sexuality" -"description": "The following are multiple choice questions (with answers) about human\ - \ sexuality.\n\n" -"group": "mmlu_alt_ov_07a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_human_sexuality" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_international_law.yaml deleted file mode 100644 index 0cea2709..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_international_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "international_law" -"description": "The following are multiple choice questions (with answers) about international\ - \ law.\n\n" -"group": "mmlu_alt_ov_07a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_international_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_jurisprudence.yaml deleted file mode 100644 index e274b3b3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_jurisprudence.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "jurisprudence" -"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ - \n" -"group": "mmlu_alt_ov_07a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_jurisprudence" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_logical_fallacies.yaml deleted file mode 100644 index a648526f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_logical_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "logical_fallacies" -"description": "The following are multiple choice questions (with answers) about logical\ - \ fallacies.\n\n" -"group": "mmlu_alt_ov_07a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_machine_learning.yaml deleted file mode 100644 index 27f5bce7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_machine_learning.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "machine_learning" -"description": "The following are multiple choice questions (with answers) about machine\ - \ learning.\n\n" -"group": "mmlu_alt_ov_07a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_machine_learning" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_management.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_management.yaml deleted file mode 100644 index 3dfe35e0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_management.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "management" -"description": "The following are multiple choice questions (with answers) about management.\n\ - \n" -"group": "mmlu_alt_ov_07a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_management" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_marketing.yaml deleted file mode 100644 index c9bca369..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_marketing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "marketing" -"description": "The following are multiple choice questions (with answers) about marketing.\n\ - \n" -"group": "mmlu_alt_ov_07a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_marketing" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_medical_genetics.yaml deleted file mode 100644 index 72f76523..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_medical_genetics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "medical_genetics" -"description": "The following are multiple choice questions (with answers) about medical\ - \ genetics.\n\n" -"group": "mmlu_alt_ov_07a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_medical_genetics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_miscellaneous.yaml deleted file mode 100644 index 9db2879e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_miscellaneous.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "miscellaneous" -"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ - \n" -"group": "mmlu_alt_ov_07a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_miscellaneous" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_moral_disputes.yaml deleted file mode 100644 index 87022179..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_moral_disputes.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_disputes" -"description": "The following are multiple choice questions (with answers) about moral\ - \ disputes.\n\n" -"group": "mmlu_alt_ov_07a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_moral_disputes" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_moral_scenarios.yaml deleted file mode 100644 index 9adcb5ed..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_moral_scenarios.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_scenarios" -"description": "The following are multiple choice questions (with answers) about moral\ - \ scenarios.\n\n" -"group": "mmlu_alt_ov_07a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_nutrition.yaml deleted file mode 100644 index dc1b458e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_nutrition.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "nutrition" -"description": "The following are multiple choice questions (with answers) about nutrition.\n\ - \n" -"group": "mmlu_alt_ov_07a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_nutrition" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_philosophy.yaml deleted file mode 100644 index f6732247..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_philosophy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "philosophy" -"description": "The following are multiple choice questions (with answers) about philosophy.\n\ - \n" -"group": "mmlu_alt_ov_07a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_philosophy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_prehistory.yaml deleted file mode 100644 index c0b82dd0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_prehistory.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "prehistory" -"description": "The following are multiple choice questions (with answers) about prehistory.\n\ - \n" -"group": "mmlu_alt_ov_07a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_prehistory" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_professional_accounting.yaml deleted file mode 100644 index 86442333..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_professional_accounting.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_accounting" -"description": "The following are multiple choice questions (with answers) about professional\ - \ accounting.\n\n" -"group": "mmlu_alt_ov_07a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_professional_accounting" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_professional_law.yaml deleted file mode 100644 index 22923265..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_professional_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_law" -"description": "The following are multiple choice questions (with answers) about professional\ - \ law.\n\n" -"group": "mmlu_alt_ov_07a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_professional_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_professional_medicine.yaml deleted file mode 100644 index b82c6937..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_professional_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_medicine" -"description": "The following are multiple choice questions (with answers) about professional\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_07a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_professional_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_professional_psychology.yaml deleted file mode 100644 index 7099ca98..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_professional_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_psychology" -"description": "The following are multiple choice questions (with answers) about professional\ - \ psychology.\n\n" -"group": "mmlu_alt_ov_07a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_professional_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_public_relations.yaml deleted file mode 100644 index 62cae6ca..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_public_relations.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "public_relations" -"description": "The following are multiple choice questions (with answers) about public\ - \ relations.\n\n" -"group": "mmlu_alt_ov_07a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_public_relations" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_security_studies.yaml deleted file mode 100644 index 9934c868..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_security_studies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "security_studies" -"description": "The following are multiple choice questions (with answers) about security\ - \ studies.\n\n" -"group": "mmlu_alt_ov_07a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_security_studies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_sociology.yaml deleted file mode 100644 index fe3c7c4f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_sociology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sociology" -"description": "The following are multiple choice questions (with answers) about sociology.\n\ - \n" -"group": "mmlu_alt_ov_07a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_sociology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_us_foreign_policy.yaml deleted file mode 100644 index 34c3a507..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_us_foreign_policy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "us_foreign_policy" -"description": "The following are multiple choice questions (with answers) about us\ - \ foreign policy.\n\n" -"group": "mmlu_alt_ov_07a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_virology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_virology.yaml deleted file mode 100644 index fcca39be..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_virology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "virology" -"description": "The following are multiple choice questions (with answers) about virology.\n\ - \n" -"group": "mmlu_alt_ov_07a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_virology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_world_religions.yaml deleted file mode 100644 index 1285fe75..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/a/mmlu_world_religions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "world_religions" -"description": "The following are multiple choice questions (with answers) about world\ - \ religions.\n\n" -"group": "mmlu_alt_ov_07a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07a_world_religions" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/_mmlu.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/_mmlu.yaml deleted file mode 100644 index 5a289391..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/_mmlu.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: mmlu_alt_ov_07b -task: - - mmlu_alt_ov_07b_stem - - mmlu_alt_ov_07b_other - - mmlu_alt_ov_07b_social_sciences - - mmlu_alt_ov_07b_humanities diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/_template_yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/_template_yaml deleted file mode 100644 index 344e8b49..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/_template_yaml +++ /dev/null @@ -1,11 +0,0 @@ -dataset_path: hails/mmlu_no_train -test_split: test -fewshot_split: dev -output_type: multiple_choice -doc_to_text: !function ../../../styles.template_07 -doc_to_choice: !function ../../../styles.choice_07b -doc_to_target: answer -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_abstract_algebra.yaml deleted file mode 100644 index 547cec13..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_abstract_algebra.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "abstract_algebra" -"description": "The following are multiple choice questions (with answers) about abstract\ - \ algebra.\n\n" -"group": "mmlu_alt_ov_07b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_anatomy.yaml deleted file mode 100644 index 449d874d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_anatomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "anatomy" -"description": "The following are multiple choice questions (with answers) about anatomy.\n\ - \n" -"group": "mmlu_alt_ov_07b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_anatomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_astronomy.yaml deleted file mode 100644 index e1db708b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_astronomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "astronomy" -"description": "The following are multiple choice questions (with answers) about astronomy.\n\ - \n" -"group": "mmlu_alt_ov_07b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_astronomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_business_ethics.yaml deleted file mode 100644 index d3625651..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_business_ethics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "business_ethics" -"description": "The following are multiple choice questions (with answers) about business\ - \ ethics.\n\n" -"group": "mmlu_alt_ov_07b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_business_ethics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_clinical_knowledge.yaml deleted file mode 100644 index 3efeccc0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_clinical_knowledge.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "clinical_knowledge" -"description": "The following are multiple choice questions (with answers) about clinical\ - \ knowledge.\n\n" -"group": "mmlu_alt_ov_07b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_college_biology.yaml deleted file mode 100644 index d363aa61..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_college_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_biology" -"description": "The following are multiple choice questions (with answers) about college\ - \ biology.\n\n" -"group": "mmlu_alt_ov_07b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_college_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_college_chemistry.yaml deleted file mode 100644 index 4531065c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_college_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_chemistry" -"description": "The following are multiple choice questions (with answers) about college\ - \ chemistry.\n\n" -"group": "mmlu_alt_ov_07b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_college_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_college_computer_science.yaml deleted file mode 100644 index 61f6f3fe..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_college_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_computer_science" -"description": "The following are multiple choice questions (with answers) about college\ - \ computer science.\n\n" -"group": "mmlu_alt_ov_07b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_college_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_college_mathematics.yaml deleted file mode 100644 index 272afa30..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_college_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_mathematics" -"description": "The following are multiple choice questions (with answers) about college\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_07b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_college_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_college_medicine.yaml deleted file mode 100644 index 7a57600b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_college_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_medicine" -"description": "The following are multiple choice questions (with answers) about college\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_07b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_college_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_college_physics.yaml deleted file mode 100644 index 1b1a4fcd..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_college_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_physics" -"description": "The following are multiple choice questions (with answers) about college\ - \ physics.\n\n" -"group": "mmlu_alt_ov_07b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_college_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_computer_security.yaml deleted file mode 100644 index a5c573a7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_computer_security.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "computer_security" -"description": "The following are multiple choice questions (with answers) about computer\ - \ security.\n\n" -"group": "mmlu_alt_ov_07b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_computer_security" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_conceptual_physics.yaml deleted file mode 100644 index 5a26c2be..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_conceptual_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "conceptual_physics" -"description": "The following are multiple choice questions (with answers) about conceptual\ - \ physics.\n\n" -"group": "mmlu_alt_ov_07b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_econometrics.yaml deleted file mode 100644 index 232a53b0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_econometrics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "econometrics" -"description": "The following are multiple choice questions (with answers) about econometrics.\n\ - \n" -"group": "mmlu_alt_ov_07b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_econometrics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_electrical_engineering.yaml deleted file mode 100644 index 604ff995..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_electrical_engineering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "electrical_engineering" -"description": "The following are multiple choice questions (with answers) about electrical\ - \ engineering.\n\n" -"group": "mmlu_alt_ov_07b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_elementary_mathematics.yaml deleted file mode 100644 index 1fdb87d9..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_elementary_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "elementary_mathematics" -"description": "The following are multiple choice questions (with answers) about elementary\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_07b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_formal_logic.yaml deleted file mode 100644 index 67954527..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_formal_logic.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_logic" -"description": "The following are multiple choice questions (with answers) about formal\ - \ logic.\n\n" -"group": "mmlu_alt_ov_07b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_formal_logic" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_global_facts.yaml deleted file mode 100644 index 2a420c03..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_global_facts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "global_facts" -"description": "The following are multiple choice questions (with answers) about global\ - \ facts.\n\n" -"group": "mmlu_alt_ov_07b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_global_facts" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_biology.yaml deleted file mode 100644 index 168b8ee5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_biology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school biology.\n\n" -"group": "mmlu_alt_ov_07b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_high_school_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_chemistry.yaml deleted file mode 100644 index bbb34bc6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_chemistry" -"description": "The following are multiple choice questions (with answers) about high\ - \ school chemistry.\n\n" -"group": "mmlu_alt_ov_07b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_computer_science.yaml deleted file mode 100644 index 5b34eb02..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_computer_science" -"description": "The following are multiple choice questions (with answers) about high\ - \ school computer science.\n\n" -"group": "mmlu_alt_ov_07b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_european_history.yaml deleted file mode 100644 index 5000f182..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_european_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_european_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school european history.\n\n" -"group": "mmlu_alt_ov_07b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_geography.yaml deleted file mode 100644 index 220495c3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_geography.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_geography" -"description": "The following are multiple choice questions (with answers) about high\ - \ school geography.\n\n" -"group": "mmlu_alt_ov_07b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_high_school_geography" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_government_and_politics.yaml deleted file mode 100644 index d8591ffa..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_government_and_politics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_government_and_politics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school government and politics.\n\n" -"group": "mmlu_alt_ov_07b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_macroeconomics.yaml deleted file mode 100644 index f7f126f9..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_macroeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_macroeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school macroeconomics.\n\n" -"group": "mmlu_alt_ov_07b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_mathematics.yaml deleted file mode 100644 index f7ca038f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_mathematics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school mathematics.\n\n" -"group": "mmlu_alt_ov_07b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_microeconomics.yaml deleted file mode 100644 index 5b4748ab..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_microeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_microeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school microeconomics.\n\n" -"group": "mmlu_alt_ov_07b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_physics.yaml deleted file mode 100644 index a7148254..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_physics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school physics.\n\n" -"group": "mmlu_alt_ov_07b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_high_school_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_psychology.yaml deleted file mode 100644 index 23f74d98..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_psychology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school psychology.\n\n" -"group": "mmlu_alt_ov_07b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_statistics.yaml deleted file mode 100644 index 5598a140..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_statistics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_statistics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school statistics.\n\n" -"group": "mmlu_alt_ov_07b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_us_history.yaml deleted file mode 100644 index 75d0cdba..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_us_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_us_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school us history.\n\n" -"group": "mmlu_alt_ov_07b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_world_history.yaml deleted file mode 100644 index 8b36f15e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_high_school_world_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_world_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school world history.\n\n" -"group": "mmlu_alt_ov_07b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_human_aging.yaml deleted file mode 100644 index e8657799..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_human_aging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_aging" -"description": "The following are multiple choice questions (with answers) about human\ - \ aging.\n\n" -"group": "mmlu_alt_ov_07b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_human_aging" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_human_sexuality.yaml deleted file mode 100644 index eb8a55f3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_human_sexuality.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_sexuality" -"description": "The following are multiple choice questions (with answers) about human\ - \ sexuality.\n\n" -"group": "mmlu_alt_ov_07b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_human_sexuality" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_international_law.yaml deleted file mode 100644 index 7c141de2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_international_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "international_law" -"description": "The following are multiple choice questions (with answers) about international\ - \ law.\n\n" -"group": "mmlu_alt_ov_07b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_international_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_jurisprudence.yaml deleted file mode 100644 index 653a8908..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_jurisprudence.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "jurisprudence" -"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ - \n" -"group": "mmlu_alt_ov_07b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_jurisprudence" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_logical_fallacies.yaml deleted file mode 100644 index 585affca..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_logical_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "logical_fallacies" -"description": "The following are multiple choice questions (with answers) about logical\ - \ fallacies.\n\n" -"group": "mmlu_alt_ov_07b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_machine_learning.yaml deleted file mode 100644 index 309b04f6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_machine_learning.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "machine_learning" -"description": "The following are multiple choice questions (with answers) about machine\ - \ learning.\n\n" -"group": "mmlu_alt_ov_07b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_machine_learning" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_management.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_management.yaml deleted file mode 100644 index 5c931d2c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_management.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "management" -"description": "The following are multiple choice questions (with answers) about management.\n\ - \n" -"group": "mmlu_alt_ov_07b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_management" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_marketing.yaml deleted file mode 100644 index 66c302b4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_marketing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "marketing" -"description": "The following are multiple choice questions (with answers) about marketing.\n\ - \n" -"group": "mmlu_alt_ov_07b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_marketing" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_medical_genetics.yaml deleted file mode 100644 index 98f1baa7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_medical_genetics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "medical_genetics" -"description": "The following are multiple choice questions (with answers) about medical\ - \ genetics.\n\n" -"group": "mmlu_alt_ov_07b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_medical_genetics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_miscellaneous.yaml deleted file mode 100644 index 8a0e7f84..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_miscellaneous.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "miscellaneous" -"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ - \n" -"group": "mmlu_alt_ov_07b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_miscellaneous" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_moral_disputes.yaml deleted file mode 100644 index 2f0acfb7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_moral_disputes.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_disputes" -"description": "The following are multiple choice questions (with answers) about moral\ - \ disputes.\n\n" -"group": "mmlu_alt_ov_07b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_moral_disputes" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_moral_scenarios.yaml deleted file mode 100644 index 17913ee9..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_moral_scenarios.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_scenarios" -"description": "The following are multiple choice questions (with answers) about moral\ - \ scenarios.\n\n" -"group": "mmlu_alt_ov_07b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_nutrition.yaml deleted file mode 100644 index 697b6654..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_nutrition.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "nutrition" -"description": "The following are multiple choice questions (with answers) about nutrition.\n\ - \n" -"group": "mmlu_alt_ov_07b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_nutrition" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_philosophy.yaml deleted file mode 100644 index 0fdccc40..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_philosophy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "philosophy" -"description": "The following are multiple choice questions (with answers) about philosophy.\n\ - \n" -"group": "mmlu_alt_ov_07b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_philosophy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_prehistory.yaml deleted file mode 100644 index 9d1d6a53..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_prehistory.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "prehistory" -"description": "The following are multiple choice questions (with answers) about prehistory.\n\ - \n" -"group": "mmlu_alt_ov_07b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_prehistory" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_professional_accounting.yaml deleted file mode 100644 index 1499c6c5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_professional_accounting.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_accounting" -"description": "The following are multiple choice questions (with answers) about professional\ - \ accounting.\n\n" -"group": "mmlu_alt_ov_07b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_professional_accounting" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_professional_law.yaml deleted file mode 100644 index a90140dc..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_professional_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_law" -"description": "The following are multiple choice questions (with answers) about professional\ - \ law.\n\n" -"group": "mmlu_alt_ov_07b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_professional_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_professional_medicine.yaml deleted file mode 100644 index ad89f65a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_professional_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_medicine" -"description": "The following are multiple choice questions (with answers) about professional\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_07b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_professional_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_professional_psychology.yaml deleted file mode 100644 index 0fe83f29..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_professional_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_psychology" -"description": "The following are multiple choice questions (with answers) about professional\ - \ psychology.\n\n" -"group": "mmlu_alt_ov_07b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_professional_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_public_relations.yaml deleted file mode 100644 index a92339b5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_public_relations.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "public_relations" -"description": "The following are multiple choice questions (with answers) about public\ - \ relations.\n\n" -"group": "mmlu_alt_ov_07b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_public_relations" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_security_studies.yaml deleted file mode 100644 index f03ac543..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_security_studies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "security_studies" -"description": "The following are multiple choice questions (with answers) about security\ - \ studies.\n\n" -"group": "mmlu_alt_ov_07b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_security_studies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_sociology.yaml deleted file mode 100644 index 20c98b44..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_sociology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sociology" -"description": "The following are multiple choice questions (with answers) about sociology.\n\ - \n" -"group": "mmlu_alt_ov_07b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_sociology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_us_foreign_policy.yaml deleted file mode 100644 index 4e9f9a59..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_us_foreign_policy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "us_foreign_policy" -"description": "The following are multiple choice questions (with answers) about us\ - \ foreign policy.\n\n" -"group": "mmlu_alt_ov_07b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_virology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_virology.yaml deleted file mode 100644 index 4e238d2c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_virology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "virology" -"description": "The following are multiple choice questions (with answers) about virology.\n\ - \n" -"group": "mmlu_alt_ov_07b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_virology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_world_religions.yaml deleted file mode 100644 index 5698c38f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/b/mmlu_world_religions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "world_religions" -"description": "The following are multiple choice questions (with answers) about world\ - \ religions.\n\n" -"group": "mmlu_alt_ov_07b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07b_world_religions" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/_mmlu.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/_mmlu.yaml deleted file mode 100644 index df79e5c8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/_mmlu.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: mmlu_alt_ov_07c -task: - - mmlu_alt_ov_07c_stem - - mmlu_alt_ov_07c_other - - mmlu_alt_ov_07c_social_sciences - - mmlu_alt_ov_07c_humanities diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/_template_yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/_template_yaml deleted file mode 100644 index e92a33eb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/_template_yaml +++ /dev/null @@ -1,11 +0,0 @@ -dataset_path: hails/mmlu_no_train -test_split: test -fewshot_split: dev -output_type: multiple_choice -doc_to_text: !function ../../../styles.template_07 -doc_to_choice: !function ../../../styles.choice_07c -doc_to_target: answer -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_abstract_algebra.yaml deleted file mode 100644 index 686388ce..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_abstract_algebra.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "abstract_algebra" -"description": "The following are multiple choice questions (with answers) about abstract\ - \ algebra.\n\n" -"group": "mmlu_alt_ov_07c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_anatomy.yaml deleted file mode 100644 index c9313eaa..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_anatomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "anatomy" -"description": "The following are multiple choice questions (with answers) about anatomy.\n\ - \n" -"group": "mmlu_alt_ov_07c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_anatomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_astronomy.yaml deleted file mode 100644 index 47b67045..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_astronomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "astronomy" -"description": "The following are multiple choice questions (with answers) about astronomy.\n\ - \n" -"group": "mmlu_alt_ov_07c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_astronomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_business_ethics.yaml deleted file mode 100644 index d3097b9c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_business_ethics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "business_ethics" -"description": "The following are multiple choice questions (with answers) about business\ - \ ethics.\n\n" -"group": "mmlu_alt_ov_07c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_business_ethics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_clinical_knowledge.yaml deleted file mode 100644 index 743516b6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_clinical_knowledge.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "clinical_knowledge" -"description": "The following are multiple choice questions (with answers) about clinical\ - \ knowledge.\n\n" -"group": "mmlu_alt_ov_07c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_college_biology.yaml deleted file mode 100644 index d79b192e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_college_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_biology" -"description": "The following are multiple choice questions (with answers) about college\ - \ biology.\n\n" -"group": "mmlu_alt_ov_07c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_college_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_college_chemistry.yaml deleted file mode 100644 index 6c082e4d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_college_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_chemistry" -"description": "The following are multiple choice questions (with answers) about college\ - \ chemistry.\n\n" -"group": "mmlu_alt_ov_07c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_college_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_college_computer_science.yaml deleted file mode 100644 index a8566fed..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_college_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_computer_science" -"description": "The following are multiple choice questions (with answers) about college\ - \ computer science.\n\n" -"group": "mmlu_alt_ov_07c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_college_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_college_mathematics.yaml deleted file mode 100644 index b765635f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_college_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_mathematics" -"description": "The following are multiple choice questions (with answers) about college\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_07c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_college_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_college_medicine.yaml deleted file mode 100644 index 90eb026a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_college_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_medicine" -"description": "The following are multiple choice questions (with answers) about college\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_07c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_college_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_college_physics.yaml deleted file mode 100644 index 311a6073..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_college_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_physics" -"description": "The following are multiple choice questions (with answers) about college\ - \ physics.\n\n" -"group": "mmlu_alt_ov_07c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_college_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_computer_security.yaml deleted file mode 100644 index f63dccbd..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_computer_security.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "computer_security" -"description": "The following are multiple choice questions (with answers) about computer\ - \ security.\n\n" -"group": "mmlu_alt_ov_07c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_computer_security" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_conceptual_physics.yaml deleted file mode 100644 index ef2a5eac..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_conceptual_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "conceptual_physics" -"description": "The following are multiple choice questions (with answers) about conceptual\ - \ physics.\n\n" -"group": "mmlu_alt_ov_07c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_econometrics.yaml deleted file mode 100644 index 2c317710..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_econometrics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "econometrics" -"description": "The following are multiple choice questions (with answers) about econometrics.\n\ - \n" -"group": "mmlu_alt_ov_07c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_econometrics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_electrical_engineering.yaml deleted file mode 100644 index ce3b5fed..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_electrical_engineering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "electrical_engineering" -"description": "The following are multiple choice questions (with answers) about electrical\ - \ engineering.\n\n" -"group": "mmlu_alt_ov_07c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_elementary_mathematics.yaml deleted file mode 100644 index 86dcd1ce..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_elementary_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "elementary_mathematics" -"description": "The following are multiple choice questions (with answers) about elementary\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_07c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_formal_logic.yaml deleted file mode 100644 index ebb40523..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_formal_logic.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_logic" -"description": "The following are multiple choice questions (with answers) about formal\ - \ logic.\n\n" -"group": "mmlu_alt_ov_07c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_formal_logic" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_global_facts.yaml deleted file mode 100644 index e9cfc2ee..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_global_facts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "global_facts" -"description": "The following are multiple choice questions (with answers) about global\ - \ facts.\n\n" -"group": "mmlu_alt_ov_07c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_global_facts" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_biology.yaml deleted file mode 100644 index c73d11f4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_biology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school biology.\n\n" -"group": "mmlu_alt_ov_07c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_high_school_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_chemistry.yaml deleted file mode 100644 index b5063b3f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_chemistry" -"description": "The following are multiple choice questions (with answers) about high\ - \ school chemistry.\n\n" -"group": "mmlu_alt_ov_07c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_computer_science.yaml deleted file mode 100644 index b3f3f7b6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_computer_science" -"description": "The following are multiple choice questions (with answers) about high\ - \ school computer science.\n\n" -"group": "mmlu_alt_ov_07c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_european_history.yaml deleted file mode 100644 index 6a3647fb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_european_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_european_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school european history.\n\n" -"group": "mmlu_alt_ov_07c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_geography.yaml deleted file mode 100644 index fc022f25..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_geography.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_geography" -"description": "The following are multiple choice questions (with answers) about high\ - \ school geography.\n\n" -"group": "mmlu_alt_ov_07c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_high_school_geography" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_government_and_politics.yaml deleted file mode 100644 index 9a205499..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_government_and_politics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_government_and_politics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school government and politics.\n\n" -"group": "mmlu_alt_ov_07c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_macroeconomics.yaml deleted file mode 100644 index 84125faa..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_macroeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_macroeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school macroeconomics.\n\n" -"group": "mmlu_alt_ov_07c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_mathematics.yaml deleted file mode 100644 index 147f21fa..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_mathematics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school mathematics.\n\n" -"group": "mmlu_alt_ov_07c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_microeconomics.yaml deleted file mode 100644 index 3d58ea11..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_microeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_microeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school microeconomics.\n\n" -"group": "mmlu_alt_ov_07c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_physics.yaml deleted file mode 100644 index 59ee2a83..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_physics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school physics.\n\n" -"group": "mmlu_alt_ov_07c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_high_school_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_psychology.yaml deleted file mode 100644 index aa344a82..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_psychology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school psychology.\n\n" -"group": "mmlu_alt_ov_07c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_statistics.yaml deleted file mode 100644 index 689675fb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_statistics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_statistics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school statistics.\n\n" -"group": "mmlu_alt_ov_07c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_us_history.yaml deleted file mode 100644 index 142e6294..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_us_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_us_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school us history.\n\n" -"group": "mmlu_alt_ov_07c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_world_history.yaml deleted file mode 100644 index d2113b49..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_high_school_world_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_world_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school world history.\n\n" -"group": "mmlu_alt_ov_07c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_human_aging.yaml deleted file mode 100644 index 22fdd806..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_human_aging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_aging" -"description": "The following are multiple choice questions (with answers) about human\ - \ aging.\n\n" -"group": "mmlu_alt_ov_07c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_human_aging" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_human_sexuality.yaml deleted file mode 100644 index 93d7fb13..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_human_sexuality.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_sexuality" -"description": "The following are multiple choice questions (with answers) about human\ - \ sexuality.\n\n" -"group": "mmlu_alt_ov_07c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_human_sexuality" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_international_law.yaml deleted file mode 100644 index acf5f6f8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_international_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "international_law" -"description": "The following are multiple choice questions (with answers) about international\ - \ law.\n\n" -"group": "mmlu_alt_ov_07c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_international_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_jurisprudence.yaml deleted file mode 100644 index 3f77c137..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_jurisprudence.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "jurisprudence" -"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ - \n" -"group": "mmlu_alt_ov_07c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_jurisprudence" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_logical_fallacies.yaml deleted file mode 100644 index 139b184d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_logical_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "logical_fallacies" -"description": "The following are multiple choice questions (with answers) about logical\ - \ fallacies.\n\n" -"group": "mmlu_alt_ov_07c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_machine_learning.yaml deleted file mode 100644 index b8559c91..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_machine_learning.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "machine_learning" -"description": "The following are multiple choice questions (with answers) about machine\ - \ learning.\n\n" -"group": "mmlu_alt_ov_07c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_machine_learning" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_management.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_management.yaml deleted file mode 100644 index 532f6884..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_management.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "management" -"description": "The following are multiple choice questions (with answers) about management.\n\ - \n" -"group": "mmlu_alt_ov_07c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_management" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_marketing.yaml deleted file mode 100644 index 5ce07359..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_marketing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "marketing" -"description": "The following are multiple choice questions (with answers) about marketing.\n\ - \n" -"group": "mmlu_alt_ov_07c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_marketing" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_medical_genetics.yaml deleted file mode 100644 index fca68ad1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_medical_genetics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "medical_genetics" -"description": "The following are multiple choice questions (with answers) about medical\ - \ genetics.\n\n" -"group": "mmlu_alt_ov_07c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_medical_genetics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_miscellaneous.yaml deleted file mode 100644 index ca70e3be..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_miscellaneous.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "miscellaneous" -"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ - \n" -"group": "mmlu_alt_ov_07c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_miscellaneous" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_moral_disputes.yaml deleted file mode 100644 index e79edbdd..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_moral_disputes.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_disputes" -"description": "The following are multiple choice questions (with answers) about moral\ - \ disputes.\n\n" -"group": "mmlu_alt_ov_07c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_moral_disputes" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_moral_scenarios.yaml deleted file mode 100644 index bf0d4b52..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_moral_scenarios.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_scenarios" -"description": "The following are multiple choice questions (with answers) about moral\ - \ scenarios.\n\n" -"group": "mmlu_alt_ov_07c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_nutrition.yaml deleted file mode 100644 index 46c5c29b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_nutrition.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "nutrition" -"description": "The following are multiple choice questions (with answers) about nutrition.\n\ - \n" -"group": "mmlu_alt_ov_07c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_nutrition" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_philosophy.yaml deleted file mode 100644 index 12f53153..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_philosophy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "philosophy" -"description": "The following are multiple choice questions (with answers) about philosophy.\n\ - \n" -"group": "mmlu_alt_ov_07c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_philosophy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_prehistory.yaml deleted file mode 100644 index 9f6002af..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_prehistory.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "prehistory" -"description": "The following are multiple choice questions (with answers) about prehistory.\n\ - \n" -"group": "mmlu_alt_ov_07c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_prehistory" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_professional_accounting.yaml deleted file mode 100644 index 6c2f6286..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_professional_accounting.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_accounting" -"description": "The following are multiple choice questions (with answers) about professional\ - \ accounting.\n\n" -"group": "mmlu_alt_ov_07c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_professional_accounting" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_professional_law.yaml deleted file mode 100644 index 709ca844..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_professional_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_law" -"description": "The following are multiple choice questions (with answers) about professional\ - \ law.\n\n" -"group": "mmlu_alt_ov_07c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_professional_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_professional_medicine.yaml deleted file mode 100644 index ea310279..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_professional_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_medicine" -"description": "The following are multiple choice questions (with answers) about professional\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_07c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_professional_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_professional_psychology.yaml deleted file mode 100644 index 3cbb0207..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_professional_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_psychology" -"description": "The following are multiple choice questions (with answers) about professional\ - \ psychology.\n\n" -"group": "mmlu_alt_ov_07c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_professional_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_public_relations.yaml deleted file mode 100644 index 87c9d318..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_public_relations.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "public_relations" -"description": "The following are multiple choice questions (with answers) about public\ - \ relations.\n\n" -"group": "mmlu_alt_ov_07c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_public_relations" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_security_studies.yaml deleted file mode 100644 index 757e716f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_security_studies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "security_studies" -"description": "The following are multiple choice questions (with answers) about security\ - \ studies.\n\n" -"group": "mmlu_alt_ov_07c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_security_studies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_sociology.yaml deleted file mode 100644 index 2a1244d5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_sociology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sociology" -"description": "The following are multiple choice questions (with answers) about sociology.\n\ - \n" -"group": "mmlu_alt_ov_07c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_sociology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_us_foreign_policy.yaml deleted file mode 100644 index 00effdcd..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_us_foreign_policy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "us_foreign_policy" -"description": "The following are multiple choice questions (with answers) about us\ - \ foreign policy.\n\n" -"group": "mmlu_alt_ov_07c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_virology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_virology.yaml deleted file mode 100644 index 238eb61d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_virology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "virology" -"description": "The following are multiple choice questions (with answers) about virology.\n\ - \n" -"group": "mmlu_alt_ov_07c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_virology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_world_religions.yaml deleted file mode 100644 index e1df0e03..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_07/c/mmlu_world_religions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "world_religions" -"description": "The following are multiple choice questions (with answers) about world\ - \ religions.\n\n" -"group": "mmlu_alt_ov_07c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_07c_world_religions" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/_mmlu_ov_08.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/_mmlu_ov_08.yaml deleted file mode 100644 index 378988db..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/_mmlu_ov_08.yaml +++ /dev/null @@ -1,5 +0,0 @@ -group: mmlu_alt_ov_08 -task: - - mmlu_alt_ov_08a - - mmlu_alt_ov_08b - - mmlu_alt_ov_08c diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/_mmlu.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/_mmlu.yaml deleted file mode 100644 index c62eaec2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/_mmlu.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: mmlu_alt_ov_08a -task: - - mmlu_alt_ov_08a_stem - - mmlu_alt_ov_08a_other - - mmlu_alt_ov_08a_social_sciences - - mmlu_alt_ov_08a_humanities diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/_template_yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/_template_yaml deleted file mode 100644 index 544c5697..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/_template_yaml +++ /dev/null @@ -1,11 +0,0 @@ -dataset_path: hails/mmlu_no_train -test_split: test -fewshot_split: dev -output_type: multiple_choice -doc_to_text: !function ../../../styles.template_08 -doc_to_choice: !function ../../../styles.choice_08a -doc_to_target: answer -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_abstract_algebra.yaml deleted file mode 100644 index c9d2d056..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_abstract_algebra.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "abstract_algebra" -"description": "The following are multiple choice questions (with answers) about abstract\ - \ algebra.\n\n" -"group": "mmlu_alt_ov_08a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_anatomy.yaml deleted file mode 100644 index 7981115c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_anatomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "anatomy" -"description": "The following are multiple choice questions (with answers) about anatomy.\n\ - \n" -"group": "mmlu_alt_ov_08a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_anatomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_astronomy.yaml deleted file mode 100644 index 690b3e18..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_astronomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "astronomy" -"description": "The following are multiple choice questions (with answers) about astronomy.\n\ - \n" -"group": "mmlu_alt_ov_08a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_astronomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_business_ethics.yaml deleted file mode 100644 index 80304895..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_business_ethics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "business_ethics" -"description": "The following are multiple choice questions (with answers) about business\ - \ ethics.\n\n" -"group": "mmlu_alt_ov_08a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_business_ethics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_clinical_knowledge.yaml deleted file mode 100644 index fbd4943f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_clinical_knowledge.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "clinical_knowledge" -"description": "The following are multiple choice questions (with answers) about clinical\ - \ knowledge.\n\n" -"group": "mmlu_alt_ov_08a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_college_biology.yaml deleted file mode 100644 index 9779430a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_college_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_biology" -"description": "The following are multiple choice questions (with answers) about college\ - \ biology.\n\n" -"group": "mmlu_alt_ov_08a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_college_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_college_chemistry.yaml deleted file mode 100644 index a87d02a6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_college_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_chemistry" -"description": "The following are multiple choice questions (with answers) about college\ - \ chemistry.\n\n" -"group": "mmlu_alt_ov_08a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_college_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_college_computer_science.yaml deleted file mode 100644 index c65f4558..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_college_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_computer_science" -"description": "The following are multiple choice questions (with answers) about college\ - \ computer science.\n\n" -"group": "mmlu_alt_ov_08a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_college_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_college_mathematics.yaml deleted file mode 100644 index 298c9f74..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_college_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_mathematics" -"description": "The following are multiple choice questions (with answers) about college\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_08a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_college_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_college_medicine.yaml deleted file mode 100644 index 4efc1b50..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_college_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_medicine" -"description": "The following are multiple choice questions (with answers) about college\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_08a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_college_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_college_physics.yaml deleted file mode 100644 index 13dda481..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_college_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_physics" -"description": "The following are multiple choice questions (with answers) about college\ - \ physics.\n\n" -"group": "mmlu_alt_ov_08a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_college_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_computer_security.yaml deleted file mode 100644 index deab7c15..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_computer_security.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "computer_security" -"description": "The following are multiple choice questions (with answers) about computer\ - \ security.\n\n" -"group": "mmlu_alt_ov_08a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_computer_security" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_conceptual_physics.yaml deleted file mode 100644 index 04abf253..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_conceptual_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "conceptual_physics" -"description": "The following are multiple choice questions (with answers) about conceptual\ - \ physics.\n\n" -"group": "mmlu_alt_ov_08a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_econometrics.yaml deleted file mode 100644 index 3ea7b4eb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_econometrics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "econometrics" -"description": "The following are multiple choice questions (with answers) about econometrics.\n\ - \n" -"group": "mmlu_alt_ov_08a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_econometrics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_electrical_engineering.yaml deleted file mode 100644 index 8f0cc66b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_electrical_engineering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "electrical_engineering" -"description": "The following are multiple choice questions (with answers) about electrical\ - \ engineering.\n\n" -"group": "mmlu_alt_ov_08a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_elementary_mathematics.yaml deleted file mode 100644 index b9f10a84..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_elementary_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "elementary_mathematics" -"description": "The following are multiple choice questions (with answers) about elementary\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_08a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_formal_logic.yaml deleted file mode 100644 index ba76ba4b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_formal_logic.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_logic" -"description": "The following are multiple choice questions (with answers) about formal\ - \ logic.\n\n" -"group": "mmlu_alt_ov_08a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_formal_logic" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_global_facts.yaml deleted file mode 100644 index 15e2cef9..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_global_facts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "global_facts" -"description": "The following are multiple choice questions (with answers) about global\ - \ facts.\n\n" -"group": "mmlu_alt_ov_08a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_global_facts" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_biology.yaml deleted file mode 100644 index 968d1f62..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_biology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school biology.\n\n" -"group": "mmlu_alt_ov_08a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_high_school_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_chemistry.yaml deleted file mode 100644 index cdab33e4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_chemistry" -"description": "The following are multiple choice questions (with answers) about high\ - \ school chemistry.\n\n" -"group": "mmlu_alt_ov_08a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_computer_science.yaml deleted file mode 100644 index ac659b8f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_computer_science" -"description": "The following are multiple choice questions (with answers) about high\ - \ school computer science.\n\n" -"group": "mmlu_alt_ov_08a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_european_history.yaml deleted file mode 100644 index c89afe32..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_european_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_european_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school european history.\n\n" -"group": "mmlu_alt_ov_08a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_geography.yaml deleted file mode 100644 index 97443442..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_geography.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_geography" -"description": "The following are multiple choice questions (with answers) about high\ - \ school geography.\n\n" -"group": "mmlu_alt_ov_08a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_high_school_geography" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_government_and_politics.yaml deleted file mode 100644 index 37d4c369..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_government_and_politics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_government_and_politics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school government and politics.\n\n" -"group": "mmlu_alt_ov_08a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_macroeconomics.yaml deleted file mode 100644 index fe3c4dae..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_macroeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_macroeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school macroeconomics.\n\n" -"group": "mmlu_alt_ov_08a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_mathematics.yaml deleted file mode 100644 index 5de10c18..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_mathematics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school mathematics.\n\n" -"group": "mmlu_alt_ov_08a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_microeconomics.yaml deleted file mode 100644 index 5eba0428..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_microeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_microeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school microeconomics.\n\n" -"group": "mmlu_alt_ov_08a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_physics.yaml deleted file mode 100644 index a61f7c97..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_physics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school physics.\n\n" -"group": "mmlu_alt_ov_08a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_high_school_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_psychology.yaml deleted file mode 100644 index 88bf97c0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_psychology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school psychology.\n\n" -"group": "mmlu_alt_ov_08a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_statistics.yaml deleted file mode 100644 index 1b0cb33c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_statistics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_statistics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school statistics.\n\n" -"group": "mmlu_alt_ov_08a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_us_history.yaml deleted file mode 100644 index 277c86b7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_us_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_us_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school us history.\n\n" -"group": "mmlu_alt_ov_08a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_world_history.yaml deleted file mode 100644 index d223949a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_high_school_world_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_world_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school world history.\n\n" -"group": "mmlu_alt_ov_08a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_human_aging.yaml deleted file mode 100644 index 2cfbfef6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_human_aging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_aging" -"description": "The following are multiple choice questions (with answers) about human\ - \ aging.\n\n" -"group": "mmlu_alt_ov_08a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_human_aging" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_human_sexuality.yaml deleted file mode 100644 index 0867836d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_human_sexuality.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_sexuality" -"description": "The following are multiple choice questions (with answers) about human\ - \ sexuality.\n\n" -"group": "mmlu_alt_ov_08a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_human_sexuality" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_international_law.yaml deleted file mode 100644 index 31362f29..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_international_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "international_law" -"description": "The following are multiple choice questions (with answers) about international\ - \ law.\n\n" -"group": "mmlu_alt_ov_08a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_international_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_jurisprudence.yaml deleted file mode 100644 index fcb7b715..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_jurisprudence.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "jurisprudence" -"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ - \n" -"group": "mmlu_alt_ov_08a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_jurisprudence" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_logical_fallacies.yaml deleted file mode 100644 index 6e5e242c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_logical_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "logical_fallacies" -"description": "The following are multiple choice questions (with answers) about logical\ - \ fallacies.\n\n" -"group": "mmlu_alt_ov_08a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_machine_learning.yaml deleted file mode 100644 index 67da0afd..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_machine_learning.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "machine_learning" -"description": "The following are multiple choice questions (with answers) about machine\ - \ learning.\n\n" -"group": "mmlu_alt_ov_08a_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_machine_learning" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_management.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_management.yaml deleted file mode 100644 index fc962a53..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_management.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "management" -"description": "The following are multiple choice questions (with answers) about management.\n\ - \n" -"group": "mmlu_alt_ov_08a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_management" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_marketing.yaml deleted file mode 100644 index bfff977d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_marketing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "marketing" -"description": "The following are multiple choice questions (with answers) about marketing.\n\ - \n" -"group": "mmlu_alt_ov_08a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_marketing" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_medical_genetics.yaml deleted file mode 100644 index 5d68da86..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_medical_genetics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "medical_genetics" -"description": "The following are multiple choice questions (with answers) about medical\ - \ genetics.\n\n" -"group": "mmlu_alt_ov_08a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_medical_genetics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_miscellaneous.yaml deleted file mode 100644 index c131112d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_miscellaneous.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "miscellaneous" -"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ - \n" -"group": "mmlu_alt_ov_08a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_miscellaneous" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_moral_disputes.yaml deleted file mode 100644 index d7daf045..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_moral_disputes.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_disputes" -"description": "The following are multiple choice questions (with answers) about moral\ - \ disputes.\n\n" -"group": "mmlu_alt_ov_08a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_moral_disputes" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_moral_scenarios.yaml deleted file mode 100644 index 848b4cb1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_moral_scenarios.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_scenarios" -"description": "The following are multiple choice questions (with answers) about moral\ - \ scenarios.\n\n" -"group": "mmlu_alt_ov_08a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_nutrition.yaml deleted file mode 100644 index 5f3dd18d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_nutrition.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "nutrition" -"description": "The following are multiple choice questions (with answers) about nutrition.\n\ - \n" -"group": "mmlu_alt_ov_08a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_nutrition" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_philosophy.yaml deleted file mode 100644 index 099f15f0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_philosophy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "philosophy" -"description": "The following are multiple choice questions (with answers) about philosophy.\n\ - \n" -"group": "mmlu_alt_ov_08a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_philosophy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_prehistory.yaml deleted file mode 100644 index 02bbfa61..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_prehistory.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "prehistory" -"description": "The following are multiple choice questions (with answers) about prehistory.\n\ - \n" -"group": "mmlu_alt_ov_08a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_prehistory" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_professional_accounting.yaml deleted file mode 100644 index b39e93c6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_professional_accounting.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_accounting" -"description": "The following are multiple choice questions (with answers) about professional\ - \ accounting.\n\n" -"group": "mmlu_alt_ov_08a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_professional_accounting" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_professional_law.yaml deleted file mode 100644 index 7b39ebf9..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_professional_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_law" -"description": "The following are multiple choice questions (with answers) about professional\ - \ law.\n\n" -"group": "mmlu_alt_ov_08a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_professional_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_professional_medicine.yaml deleted file mode 100644 index 4c70ad68..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_professional_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_medicine" -"description": "The following are multiple choice questions (with answers) about professional\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_08a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_professional_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_professional_psychology.yaml deleted file mode 100644 index fb06ce62..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_professional_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_psychology" -"description": "The following are multiple choice questions (with answers) about professional\ - \ psychology.\n\n" -"group": "mmlu_alt_ov_08a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_professional_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_public_relations.yaml deleted file mode 100644 index 3fe438c2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_public_relations.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "public_relations" -"description": "The following are multiple choice questions (with answers) about public\ - \ relations.\n\n" -"group": "mmlu_alt_ov_08a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_public_relations" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_security_studies.yaml deleted file mode 100644 index 0794c491..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_security_studies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "security_studies" -"description": "The following are multiple choice questions (with answers) about security\ - \ studies.\n\n" -"group": "mmlu_alt_ov_08a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_security_studies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_sociology.yaml deleted file mode 100644 index 1f9fc02a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_sociology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sociology" -"description": "The following are multiple choice questions (with answers) about sociology.\n\ - \n" -"group": "mmlu_alt_ov_08a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_sociology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_us_foreign_policy.yaml deleted file mode 100644 index 77bbe90a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_us_foreign_policy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "us_foreign_policy" -"description": "The following are multiple choice questions (with answers) about us\ - \ foreign policy.\n\n" -"group": "mmlu_alt_ov_08a_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_virology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_virology.yaml deleted file mode 100644 index da2aa761..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_virology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "virology" -"description": "The following are multiple choice questions (with answers) about virology.\n\ - \n" -"group": "mmlu_alt_ov_08a_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_virology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_world_religions.yaml deleted file mode 100644 index 1d636c34..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/a/mmlu_world_religions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "world_religions" -"description": "The following are multiple choice questions (with answers) about world\ - \ religions.\n\n" -"group": "mmlu_alt_ov_08a_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08a_world_religions" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/_mmlu.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/_mmlu.yaml deleted file mode 100644 index f872f7cd..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/_mmlu.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: mmlu_alt_ov_08b -task: - - mmlu_alt_ov_08b_stem - - mmlu_alt_ov_08b_other - - mmlu_alt_ov_08b_social_sciences - - mmlu_alt_ov_08b_humanities diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/_template_yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/_template_yaml deleted file mode 100644 index fd9d94df..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/_template_yaml +++ /dev/null @@ -1,11 +0,0 @@ -dataset_path: hails/mmlu_no_train -test_split: test -fewshot_split: dev -output_type: multiple_choice -doc_to_text: !function ../../../styles.template_08 -doc_to_choice: !function ../../../styles.choice_08b -doc_to_target: answer -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_abstract_algebra.yaml deleted file mode 100644 index a0fbe9c6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_abstract_algebra.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "abstract_algebra" -"description": "The following are multiple choice questions (with answers) about abstract\ - \ algebra.\n\n" -"group": "mmlu_alt_ov_08b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_anatomy.yaml deleted file mode 100644 index 58bfd0b1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_anatomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "anatomy" -"description": "The following are multiple choice questions (with answers) about anatomy.\n\ - \n" -"group": "mmlu_alt_ov_08b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_anatomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_astronomy.yaml deleted file mode 100644 index 9eca5e1f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_astronomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "astronomy" -"description": "The following are multiple choice questions (with answers) about astronomy.\n\ - \n" -"group": "mmlu_alt_ov_08b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_astronomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_business_ethics.yaml deleted file mode 100644 index 36995b7d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_business_ethics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "business_ethics" -"description": "The following are multiple choice questions (with answers) about business\ - \ ethics.\n\n" -"group": "mmlu_alt_ov_08b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_business_ethics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_clinical_knowledge.yaml deleted file mode 100644 index 56123d3b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_clinical_knowledge.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "clinical_knowledge" -"description": "The following are multiple choice questions (with answers) about clinical\ - \ knowledge.\n\n" -"group": "mmlu_alt_ov_08b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_college_biology.yaml deleted file mode 100644 index 13e57ee3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_college_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_biology" -"description": "The following are multiple choice questions (with answers) about college\ - \ biology.\n\n" -"group": "mmlu_alt_ov_08b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_college_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_college_chemistry.yaml deleted file mode 100644 index 9734b443..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_college_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_chemistry" -"description": "The following are multiple choice questions (with answers) about college\ - \ chemistry.\n\n" -"group": "mmlu_alt_ov_08b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_college_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_college_computer_science.yaml deleted file mode 100644 index fc442741..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_college_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_computer_science" -"description": "The following are multiple choice questions (with answers) about college\ - \ computer science.\n\n" -"group": "mmlu_alt_ov_08b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_college_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_college_mathematics.yaml deleted file mode 100644 index 2d89509d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_college_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_mathematics" -"description": "The following are multiple choice questions (with answers) about college\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_08b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_college_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_college_medicine.yaml deleted file mode 100644 index 19dbfc12..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_college_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_medicine" -"description": "The following are multiple choice questions (with answers) about college\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_08b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_college_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_college_physics.yaml deleted file mode 100644 index 6d738349..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_college_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_physics" -"description": "The following are multiple choice questions (with answers) about college\ - \ physics.\n\n" -"group": "mmlu_alt_ov_08b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_college_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_computer_security.yaml deleted file mode 100644 index c02dbf14..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_computer_security.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "computer_security" -"description": "The following are multiple choice questions (with answers) about computer\ - \ security.\n\n" -"group": "mmlu_alt_ov_08b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_computer_security" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_conceptual_physics.yaml deleted file mode 100644 index 185ee5bd..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_conceptual_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "conceptual_physics" -"description": "The following are multiple choice questions (with answers) about conceptual\ - \ physics.\n\n" -"group": "mmlu_alt_ov_08b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_econometrics.yaml deleted file mode 100644 index ab318dac..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_econometrics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "econometrics" -"description": "The following are multiple choice questions (with answers) about econometrics.\n\ - \n" -"group": "mmlu_alt_ov_08b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_econometrics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_electrical_engineering.yaml deleted file mode 100644 index 1efd8a5e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_electrical_engineering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "electrical_engineering" -"description": "The following are multiple choice questions (with answers) about electrical\ - \ engineering.\n\n" -"group": "mmlu_alt_ov_08b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_elementary_mathematics.yaml deleted file mode 100644 index f36aeec4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_elementary_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "elementary_mathematics" -"description": "The following are multiple choice questions (with answers) about elementary\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_08b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_formal_logic.yaml deleted file mode 100644 index d5556b88..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_formal_logic.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_logic" -"description": "The following are multiple choice questions (with answers) about formal\ - \ logic.\n\n" -"group": "mmlu_alt_ov_08b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_formal_logic" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_global_facts.yaml deleted file mode 100644 index 75cd9512..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_global_facts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "global_facts" -"description": "The following are multiple choice questions (with answers) about global\ - \ facts.\n\n" -"group": "mmlu_alt_ov_08b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_global_facts" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_biology.yaml deleted file mode 100644 index 1474b407..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_biology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school biology.\n\n" -"group": "mmlu_alt_ov_08b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_high_school_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_chemistry.yaml deleted file mode 100644 index 060a490e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_chemistry" -"description": "The following are multiple choice questions (with answers) about high\ - \ school chemistry.\n\n" -"group": "mmlu_alt_ov_08b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_computer_science.yaml deleted file mode 100644 index 355109d2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_computer_science" -"description": "The following are multiple choice questions (with answers) about high\ - \ school computer science.\n\n" -"group": "mmlu_alt_ov_08b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_european_history.yaml deleted file mode 100644 index 360c30d8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_european_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_european_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school european history.\n\n" -"group": "mmlu_alt_ov_08b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_geography.yaml deleted file mode 100644 index 693f58ab..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_geography.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_geography" -"description": "The following are multiple choice questions (with answers) about high\ - \ school geography.\n\n" -"group": "mmlu_alt_ov_08b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_high_school_geography" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_government_and_politics.yaml deleted file mode 100644 index 999ff848..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_government_and_politics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_government_and_politics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school government and politics.\n\n" -"group": "mmlu_alt_ov_08b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_macroeconomics.yaml deleted file mode 100644 index 18d61f39..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_macroeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_macroeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school macroeconomics.\n\n" -"group": "mmlu_alt_ov_08b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_mathematics.yaml deleted file mode 100644 index f3f23469..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_mathematics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school mathematics.\n\n" -"group": "mmlu_alt_ov_08b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_microeconomics.yaml deleted file mode 100644 index c4bc1bbb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_microeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_microeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school microeconomics.\n\n" -"group": "mmlu_alt_ov_08b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_physics.yaml deleted file mode 100644 index 0659a800..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_physics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school physics.\n\n" -"group": "mmlu_alt_ov_08b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_high_school_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_psychology.yaml deleted file mode 100644 index 803e8b12..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_psychology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school psychology.\n\n" -"group": "mmlu_alt_ov_08b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_statistics.yaml deleted file mode 100644 index 5429f118..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_statistics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_statistics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school statistics.\n\n" -"group": "mmlu_alt_ov_08b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_us_history.yaml deleted file mode 100644 index 495f7b79..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_us_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_us_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school us history.\n\n" -"group": "mmlu_alt_ov_08b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_world_history.yaml deleted file mode 100644 index 3dab44fa..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_high_school_world_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_world_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school world history.\n\n" -"group": "mmlu_alt_ov_08b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_human_aging.yaml deleted file mode 100644 index d5b9aee9..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_human_aging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_aging" -"description": "The following are multiple choice questions (with answers) about human\ - \ aging.\n\n" -"group": "mmlu_alt_ov_08b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_human_aging" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_human_sexuality.yaml deleted file mode 100644 index 213f13df..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_human_sexuality.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_sexuality" -"description": "The following are multiple choice questions (with answers) about human\ - \ sexuality.\n\n" -"group": "mmlu_alt_ov_08b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_human_sexuality" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_international_law.yaml deleted file mode 100644 index 2cfe6b5c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_international_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "international_law" -"description": "The following are multiple choice questions (with answers) about international\ - \ law.\n\n" -"group": "mmlu_alt_ov_08b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_international_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_jurisprudence.yaml deleted file mode 100644 index 191d570b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_jurisprudence.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "jurisprudence" -"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ - \n" -"group": "mmlu_alt_ov_08b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_jurisprudence" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_logical_fallacies.yaml deleted file mode 100644 index 331e54ab..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_logical_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "logical_fallacies" -"description": "The following are multiple choice questions (with answers) about logical\ - \ fallacies.\n\n" -"group": "mmlu_alt_ov_08b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_machine_learning.yaml deleted file mode 100644 index 9232faca..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_machine_learning.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "machine_learning" -"description": "The following are multiple choice questions (with answers) about machine\ - \ learning.\n\n" -"group": "mmlu_alt_ov_08b_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_machine_learning" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_management.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_management.yaml deleted file mode 100644 index 89cc432b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_management.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "management" -"description": "The following are multiple choice questions (with answers) about management.\n\ - \n" -"group": "mmlu_alt_ov_08b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_management" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_marketing.yaml deleted file mode 100644 index e06b0100..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_marketing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "marketing" -"description": "The following are multiple choice questions (with answers) about marketing.\n\ - \n" -"group": "mmlu_alt_ov_08b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_marketing" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_medical_genetics.yaml deleted file mode 100644 index f06299df..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_medical_genetics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "medical_genetics" -"description": "The following are multiple choice questions (with answers) about medical\ - \ genetics.\n\n" -"group": "mmlu_alt_ov_08b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_medical_genetics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_miscellaneous.yaml deleted file mode 100644 index 10456aa1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_miscellaneous.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "miscellaneous" -"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ - \n" -"group": "mmlu_alt_ov_08b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_miscellaneous" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_moral_disputes.yaml deleted file mode 100644 index 5057f1e0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_moral_disputes.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_disputes" -"description": "The following are multiple choice questions (with answers) about moral\ - \ disputes.\n\n" -"group": "mmlu_alt_ov_08b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_moral_disputes" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_moral_scenarios.yaml deleted file mode 100644 index 78ed268a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_moral_scenarios.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_scenarios" -"description": "The following are multiple choice questions (with answers) about moral\ - \ scenarios.\n\n" -"group": "mmlu_alt_ov_08b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_nutrition.yaml deleted file mode 100644 index 89bd98f0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_nutrition.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "nutrition" -"description": "The following are multiple choice questions (with answers) about nutrition.\n\ - \n" -"group": "mmlu_alt_ov_08b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_nutrition" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_philosophy.yaml deleted file mode 100644 index d8566dcd..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_philosophy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "philosophy" -"description": "The following are multiple choice questions (with answers) about philosophy.\n\ - \n" -"group": "mmlu_alt_ov_08b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_philosophy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_prehistory.yaml deleted file mode 100644 index f34e833c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_prehistory.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "prehistory" -"description": "The following are multiple choice questions (with answers) about prehistory.\n\ - \n" -"group": "mmlu_alt_ov_08b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_prehistory" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_professional_accounting.yaml deleted file mode 100644 index fabe6f80..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_professional_accounting.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_accounting" -"description": "The following are multiple choice questions (with answers) about professional\ - \ accounting.\n\n" -"group": "mmlu_alt_ov_08b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_professional_accounting" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_professional_law.yaml deleted file mode 100644 index dace16a8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_professional_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_law" -"description": "The following are multiple choice questions (with answers) about professional\ - \ law.\n\n" -"group": "mmlu_alt_ov_08b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_professional_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_professional_medicine.yaml deleted file mode 100644 index 8b49a0b5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_professional_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_medicine" -"description": "The following are multiple choice questions (with answers) about professional\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_08b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_professional_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_professional_psychology.yaml deleted file mode 100644 index 9c785893..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_professional_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_psychology" -"description": "The following are multiple choice questions (with answers) about professional\ - \ psychology.\n\n" -"group": "mmlu_alt_ov_08b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_professional_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_public_relations.yaml deleted file mode 100644 index 8401a770..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_public_relations.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "public_relations" -"description": "The following are multiple choice questions (with answers) about public\ - \ relations.\n\n" -"group": "mmlu_alt_ov_08b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_public_relations" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_security_studies.yaml deleted file mode 100644 index bc4cc164..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_security_studies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "security_studies" -"description": "The following are multiple choice questions (with answers) about security\ - \ studies.\n\n" -"group": "mmlu_alt_ov_08b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_security_studies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_sociology.yaml deleted file mode 100644 index b7124605..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_sociology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sociology" -"description": "The following are multiple choice questions (with answers) about sociology.\n\ - \n" -"group": "mmlu_alt_ov_08b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_sociology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_us_foreign_policy.yaml deleted file mode 100644 index 11fcb95f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_us_foreign_policy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "us_foreign_policy" -"description": "The following are multiple choice questions (with answers) about us\ - \ foreign policy.\n\n" -"group": "mmlu_alt_ov_08b_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_virology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_virology.yaml deleted file mode 100644 index de8b1fd0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_virology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "virology" -"description": "The following are multiple choice questions (with answers) about virology.\n\ - \n" -"group": "mmlu_alt_ov_08b_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_virology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_world_religions.yaml deleted file mode 100644 index b4d5a5d6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/b/mmlu_world_religions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "world_religions" -"description": "The following are multiple choice questions (with answers) about world\ - \ religions.\n\n" -"group": "mmlu_alt_ov_08b_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08b_world_religions" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/_mmlu.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/_mmlu.yaml deleted file mode 100644 index cd6e248b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/_mmlu.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: mmlu_alt_ov_08c -task: - - mmlu_alt_ov_08c_stem - - mmlu_alt_ov_08c_other - - mmlu_alt_ov_08c_social_sciences - - mmlu_alt_ov_08c_humanities diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/_template_yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/_template_yaml deleted file mode 100644 index fccff023..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/_template_yaml +++ /dev/null @@ -1,11 +0,0 @@ -dataset_path: hails/mmlu_no_train -test_split: test -fewshot_split: dev -output_type: multiple_choice -doc_to_text: !function ../../../styles.template_08 -doc_to_choice: !function ../../../styles.choice_08c -doc_to_target: answer -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_abstract_algebra.yaml deleted file mode 100644 index 28cad616..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_abstract_algebra.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "abstract_algebra" -"description": "The following are multiple choice questions (with answers) about abstract\ - \ algebra.\n\n" -"group": "mmlu_alt_ov_08c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_anatomy.yaml deleted file mode 100644 index 118593a2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_anatomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "anatomy" -"description": "The following are multiple choice questions (with answers) about anatomy.\n\ - \n" -"group": "mmlu_alt_ov_08c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_anatomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_astronomy.yaml deleted file mode 100644 index 11dedec1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_astronomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "astronomy" -"description": "The following are multiple choice questions (with answers) about astronomy.\n\ - \n" -"group": "mmlu_alt_ov_08c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_astronomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_business_ethics.yaml deleted file mode 100644 index 49c7bf7f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_business_ethics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "business_ethics" -"description": "The following are multiple choice questions (with answers) about business\ - \ ethics.\n\n" -"group": "mmlu_alt_ov_08c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_business_ethics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_clinical_knowledge.yaml deleted file mode 100644 index 8722c2f8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_clinical_knowledge.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "clinical_knowledge" -"description": "The following are multiple choice questions (with answers) about clinical\ - \ knowledge.\n\n" -"group": "mmlu_alt_ov_08c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_college_biology.yaml deleted file mode 100644 index fc4ccd5e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_college_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_biology" -"description": "The following are multiple choice questions (with answers) about college\ - \ biology.\n\n" -"group": "mmlu_alt_ov_08c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_college_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_college_chemistry.yaml deleted file mode 100644 index f4690aba..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_college_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_chemistry" -"description": "The following are multiple choice questions (with answers) about college\ - \ chemistry.\n\n" -"group": "mmlu_alt_ov_08c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_college_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_college_computer_science.yaml deleted file mode 100644 index 03d4edfc..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_college_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_computer_science" -"description": "The following are multiple choice questions (with answers) about college\ - \ computer science.\n\n" -"group": "mmlu_alt_ov_08c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_college_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_college_mathematics.yaml deleted file mode 100644 index 268fd6dc..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_college_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_mathematics" -"description": "The following are multiple choice questions (with answers) about college\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_08c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_college_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_college_medicine.yaml deleted file mode 100644 index eb5cd9da..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_college_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_medicine" -"description": "The following are multiple choice questions (with answers) about college\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_08c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_college_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_college_physics.yaml deleted file mode 100644 index 1da6f78a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_college_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_physics" -"description": "The following are multiple choice questions (with answers) about college\ - \ physics.\n\n" -"group": "mmlu_alt_ov_08c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_college_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_computer_security.yaml deleted file mode 100644 index cf299685..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_computer_security.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "computer_security" -"description": "The following are multiple choice questions (with answers) about computer\ - \ security.\n\n" -"group": "mmlu_alt_ov_08c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_computer_security" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_conceptual_physics.yaml deleted file mode 100644 index e0134277..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_conceptual_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "conceptual_physics" -"description": "The following are multiple choice questions (with answers) about conceptual\ - \ physics.\n\n" -"group": "mmlu_alt_ov_08c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_econometrics.yaml deleted file mode 100644 index 184e057e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_econometrics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "econometrics" -"description": "The following are multiple choice questions (with answers) about econometrics.\n\ - \n" -"group": "mmlu_alt_ov_08c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_econometrics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_electrical_engineering.yaml deleted file mode 100644 index 6a539547..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_electrical_engineering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "electrical_engineering" -"description": "The following are multiple choice questions (with answers) about electrical\ - \ engineering.\n\n" -"group": "mmlu_alt_ov_08c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_elementary_mathematics.yaml deleted file mode 100644 index 65e9f81e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_elementary_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "elementary_mathematics" -"description": "The following are multiple choice questions (with answers) about elementary\ - \ mathematics.\n\n" -"group": "mmlu_alt_ov_08c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_formal_logic.yaml deleted file mode 100644 index 21ac6a0d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_formal_logic.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_logic" -"description": "The following are multiple choice questions (with answers) about formal\ - \ logic.\n\n" -"group": "mmlu_alt_ov_08c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_formal_logic" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_global_facts.yaml deleted file mode 100644 index c0eb40ac..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_global_facts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "global_facts" -"description": "The following are multiple choice questions (with answers) about global\ - \ facts.\n\n" -"group": "mmlu_alt_ov_08c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_global_facts" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_biology.yaml deleted file mode 100644 index 37e54dd5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_biology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school biology.\n\n" -"group": "mmlu_alt_ov_08c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_high_school_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_chemistry.yaml deleted file mode 100644 index de1c42ab..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_chemistry" -"description": "The following are multiple choice questions (with answers) about high\ - \ school chemistry.\n\n" -"group": "mmlu_alt_ov_08c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_computer_science.yaml deleted file mode 100644 index b4aa54c4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_computer_science" -"description": "The following are multiple choice questions (with answers) about high\ - \ school computer science.\n\n" -"group": "mmlu_alt_ov_08c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_european_history.yaml deleted file mode 100644 index 8bc00484..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_european_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_european_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school european history.\n\n" -"group": "mmlu_alt_ov_08c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_geography.yaml deleted file mode 100644 index bac6e812..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_geography.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_geography" -"description": "The following are multiple choice questions (with answers) about high\ - \ school geography.\n\n" -"group": "mmlu_alt_ov_08c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_high_school_geography" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_government_and_politics.yaml deleted file mode 100644 index 1b9de7a3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_government_and_politics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_government_and_politics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school government and politics.\n\n" -"group": "mmlu_alt_ov_08c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_macroeconomics.yaml deleted file mode 100644 index c7036e9c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_macroeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_macroeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school macroeconomics.\n\n" -"group": "mmlu_alt_ov_08c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_mathematics.yaml deleted file mode 100644 index 6952d358..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_mathematics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school mathematics.\n\n" -"group": "mmlu_alt_ov_08c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_microeconomics.yaml deleted file mode 100644 index 13c836ca..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_microeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_microeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school microeconomics.\n\n" -"group": "mmlu_alt_ov_08c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_physics.yaml deleted file mode 100644 index 89bdcf54..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_physics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school physics.\n\n" -"group": "mmlu_alt_ov_08c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_high_school_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_psychology.yaml deleted file mode 100644 index 58b2b077..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_psychology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school psychology.\n\n" -"group": "mmlu_alt_ov_08c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_statistics.yaml deleted file mode 100644 index 811af4a1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_statistics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_statistics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school statistics.\n\n" -"group": "mmlu_alt_ov_08c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_us_history.yaml deleted file mode 100644 index 6fdc9483..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_us_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_us_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school us history.\n\n" -"group": "mmlu_alt_ov_08c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_world_history.yaml deleted file mode 100644 index f39b8330..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_high_school_world_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_world_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school world history.\n\n" -"group": "mmlu_alt_ov_08c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_human_aging.yaml deleted file mode 100644 index 5e452b51..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_human_aging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_aging" -"description": "The following are multiple choice questions (with answers) about human\ - \ aging.\n\n" -"group": "mmlu_alt_ov_08c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_human_aging" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_human_sexuality.yaml deleted file mode 100644 index 05804591..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_human_sexuality.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_sexuality" -"description": "The following are multiple choice questions (with answers) about human\ - \ sexuality.\n\n" -"group": "mmlu_alt_ov_08c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_human_sexuality" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_international_law.yaml deleted file mode 100644 index 9798c465..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_international_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "international_law" -"description": "The following are multiple choice questions (with answers) about international\ - \ law.\n\n" -"group": "mmlu_alt_ov_08c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_international_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_jurisprudence.yaml deleted file mode 100644 index 5bc1236c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_jurisprudence.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "jurisprudence" -"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ - \n" -"group": "mmlu_alt_ov_08c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_jurisprudence" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_logical_fallacies.yaml deleted file mode 100644 index 659afcc6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_logical_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "logical_fallacies" -"description": "The following are multiple choice questions (with answers) about logical\ - \ fallacies.\n\n" -"group": "mmlu_alt_ov_08c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_machine_learning.yaml deleted file mode 100644 index 34e70b76..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_machine_learning.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "machine_learning" -"description": "The following are multiple choice questions (with answers) about machine\ - \ learning.\n\n" -"group": "mmlu_alt_ov_08c_stem" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_machine_learning" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_management.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_management.yaml deleted file mode 100644 index 734aa96c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_management.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "management" -"description": "The following are multiple choice questions (with answers) about management.\n\ - \n" -"group": "mmlu_alt_ov_08c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_management" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_marketing.yaml deleted file mode 100644 index c36b4750..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_marketing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "marketing" -"description": "The following are multiple choice questions (with answers) about marketing.\n\ - \n" -"group": "mmlu_alt_ov_08c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_marketing" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_medical_genetics.yaml deleted file mode 100644 index 53d0fab7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_medical_genetics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "medical_genetics" -"description": "The following are multiple choice questions (with answers) about medical\ - \ genetics.\n\n" -"group": "mmlu_alt_ov_08c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_medical_genetics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_miscellaneous.yaml deleted file mode 100644 index 8d994dd5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_miscellaneous.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "miscellaneous" -"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ - \n" -"group": "mmlu_alt_ov_08c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_miscellaneous" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_moral_disputes.yaml deleted file mode 100644 index ac8daaf8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_moral_disputes.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_disputes" -"description": "The following are multiple choice questions (with answers) about moral\ - \ disputes.\n\n" -"group": "mmlu_alt_ov_08c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_moral_disputes" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_moral_scenarios.yaml deleted file mode 100644 index e2e5fe42..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_moral_scenarios.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_scenarios" -"description": "The following are multiple choice questions (with answers) about moral\ - \ scenarios.\n\n" -"group": "mmlu_alt_ov_08c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_nutrition.yaml deleted file mode 100644 index 4ad5a3f7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_nutrition.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "nutrition" -"description": "The following are multiple choice questions (with answers) about nutrition.\n\ - \n" -"group": "mmlu_alt_ov_08c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_nutrition" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_philosophy.yaml deleted file mode 100644 index 16994446..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_philosophy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "philosophy" -"description": "The following are multiple choice questions (with answers) about philosophy.\n\ - \n" -"group": "mmlu_alt_ov_08c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_philosophy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_prehistory.yaml deleted file mode 100644 index 23e327d1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_prehistory.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "prehistory" -"description": "The following are multiple choice questions (with answers) about prehistory.\n\ - \n" -"group": "mmlu_alt_ov_08c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_prehistory" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_professional_accounting.yaml deleted file mode 100644 index 9515d77d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_professional_accounting.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_accounting" -"description": "The following are multiple choice questions (with answers) about professional\ - \ accounting.\n\n" -"group": "mmlu_alt_ov_08c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_professional_accounting" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_professional_law.yaml deleted file mode 100644 index d6187d91..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_professional_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_law" -"description": "The following are multiple choice questions (with answers) about professional\ - \ law.\n\n" -"group": "mmlu_alt_ov_08c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_professional_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_professional_medicine.yaml deleted file mode 100644 index 00a2136c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_professional_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_medicine" -"description": "The following are multiple choice questions (with answers) about professional\ - \ medicine.\n\n" -"group": "mmlu_alt_ov_08c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_professional_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_professional_psychology.yaml deleted file mode 100644 index 898eb37e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_professional_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_psychology" -"description": "The following are multiple choice questions (with answers) about professional\ - \ psychology.\n\n" -"group": "mmlu_alt_ov_08c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_professional_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_public_relations.yaml deleted file mode 100644 index f00eb973..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_public_relations.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "public_relations" -"description": "The following are multiple choice questions (with answers) about public\ - \ relations.\n\n" -"group": "mmlu_alt_ov_08c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_public_relations" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_security_studies.yaml deleted file mode 100644 index 0ec582d6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_security_studies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "security_studies" -"description": "The following are multiple choice questions (with answers) about security\ - \ studies.\n\n" -"group": "mmlu_alt_ov_08c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_security_studies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_sociology.yaml deleted file mode 100644 index a06d4e2f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_sociology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sociology" -"description": "The following are multiple choice questions (with answers) about sociology.\n\ - \n" -"group": "mmlu_alt_ov_08c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_sociology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_us_foreign_policy.yaml deleted file mode 100644 index b48618f3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_us_foreign_policy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "us_foreign_policy" -"description": "The following are multiple choice questions (with answers) about us\ - \ foreign policy.\n\n" -"group": "mmlu_alt_ov_08c_social_sciences" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_virology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_virology.yaml deleted file mode 100644 index 8b0cfce7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_virology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "virology" -"description": "The following are multiple choice questions (with answers) about virology.\n\ - \n" -"group": "mmlu_alt_ov_08c_other" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_virology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_world_religions.yaml deleted file mode 100644 index 8ef51c00..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_08/c/mmlu_world_religions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "world_religions" -"description": "The following are multiple choice questions (with answers) about world\ - \ religions.\n\n" -"group": "mmlu_alt_ov_08c_humanities" -"include": "_template_yaml" -"task": "mmlu_alt_ov_08c_world_religions" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/_default_template_yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/_default_template_yaml deleted file mode 100644 index 4f73aa15..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/_default_template_yaml +++ /dev/null @@ -1,13 +0,0 @@ -dataset_path: hails/mmlu_no_train # a copy of `cais/mmlu` with no auxiliary_train split -test_split: test -fewshot_split: dev -fewshot_config: - sampler: first_n -output_type: multiple_choice -doc_to_text: "{{question.strip()}}" -doc_to_choice: choices -doc_to_target: answer -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/_mmlu.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/_mmlu.yaml deleted file mode 100644 index 83bce395..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/_mmlu.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: mmlu_alt_pv_01 -task: - - mmlu_alt_pv_01_stem - - mmlu_alt_pv_01_other - - mmlu_alt_pv_01_social_sciences - - mmlu_alt_pv_01_humanities diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_abstract_algebra.yaml deleted file mode 100644 index 6ecfa807..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_abstract_algebra.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "abstract_algebra" -"description": "The following are multiple choice questions (with answers) about abstract\ - \ algebra.\n\n" -"group": "mmlu_alt_pv_01_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_anatomy.yaml deleted file mode 100644 index ae8eb7a5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_anatomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "anatomy" -"description": "The following are multiple choice questions (with answers) about anatomy.\n\ - \n" -"group": "mmlu_alt_pv_01_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_anatomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_astronomy.yaml deleted file mode 100644 index 688106b8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_astronomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "astronomy" -"description": "The following are multiple choice questions (with answers) about astronomy.\n\ - \n" -"group": "mmlu_alt_pv_01_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_astronomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_business_ethics.yaml deleted file mode 100644 index 31a0f39a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_business_ethics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "business_ethics" -"description": "The following are multiple choice questions (with answers) about business\ - \ ethics.\n\n" -"group": "mmlu_alt_pv_01_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_business_ethics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_clinical_knowledge.yaml deleted file mode 100644 index b860ba1b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_clinical_knowledge.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "clinical_knowledge" -"description": "The following are multiple choice questions (with answers) about clinical\ - \ knowledge.\n\n" -"group": "mmlu_alt_pv_01_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_college_biology.yaml deleted file mode 100644 index 8c5464ca..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_college_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_biology" -"description": "The following are multiple choice questions (with answers) about college\ - \ biology.\n\n" -"group": "mmlu_alt_pv_01_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_college_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_college_chemistry.yaml deleted file mode 100644 index 8938bd03..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_college_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_chemistry" -"description": "The following are multiple choice questions (with answers) about college\ - \ chemistry.\n\n" -"group": "mmlu_alt_pv_01_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_college_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_college_computer_science.yaml deleted file mode 100644 index 9f0829a8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_college_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_computer_science" -"description": "The following are multiple choice questions (with answers) about college\ - \ computer science.\n\n" -"group": "mmlu_alt_pv_01_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_college_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_college_mathematics.yaml deleted file mode 100644 index e0fde29e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_college_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_mathematics" -"description": "The following are multiple choice questions (with answers) about college\ - \ mathematics.\n\n" -"group": "mmlu_alt_pv_01_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_college_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_college_medicine.yaml deleted file mode 100644 index 98e29dc6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_college_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_medicine" -"description": "The following are multiple choice questions (with answers) about college\ - \ medicine.\n\n" -"group": "mmlu_alt_pv_01_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_college_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_college_physics.yaml deleted file mode 100644 index a9249498..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_college_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_physics" -"description": "The following are multiple choice questions (with answers) about college\ - \ physics.\n\n" -"group": "mmlu_alt_pv_01_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_college_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_computer_security.yaml deleted file mode 100644 index fb127bad..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_computer_security.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "computer_security" -"description": "The following are multiple choice questions (with answers) about computer\ - \ security.\n\n" -"group": "mmlu_alt_pv_01_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_computer_security" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_conceptual_physics.yaml deleted file mode 100644 index 9d054a75..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_conceptual_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "conceptual_physics" -"description": "The following are multiple choice questions (with answers) about conceptual\ - \ physics.\n\n" -"group": "mmlu_alt_pv_01_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_econometrics.yaml deleted file mode 100644 index aa25237d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_econometrics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "econometrics" -"description": "The following are multiple choice questions (with answers) about econometrics.\n\ - \n" -"group": "mmlu_alt_pv_01_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_econometrics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_electrical_engineering.yaml deleted file mode 100644 index 551c1544..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_electrical_engineering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "electrical_engineering" -"description": "The following are multiple choice questions (with answers) about electrical\ - \ engineering.\n\n" -"group": "mmlu_alt_pv_01_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_elementary_mathematics.yaml deleted file mode 100644 index e8034806..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_elementary_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "elementary_mathematics" -"description": "The following are multiple choice questions (with answers) about elementary\ - \ mathematics.\n\n" -"group": "mmlu_alt_pv_01_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_formal_logic.yaml deleted file mode 100644 index 7517dd5d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_formal_logic.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_logic" -"description": "The following are multiple choice questions (with answers) about formal\ - \ logic.\n\n" -"group": "mmlu_alt_pv_01_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_formal_logic" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_global_facts.yaml deleted file mode 100644 index 15f4a14e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_global_facts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "global_facts" -"description": "The following are multiple choice questions (with answers) about global\ - \ facts.\n\n" -"group": "mmlu_alt_pv_01_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_global_facts" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_biology.yaml deleted file mode 100644 index 2d514e0b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_biology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school biology.\n\n" -"group": "mmlu_alt_pv_01_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_high_school_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_chemistry.yaml deleted file mode 100644 index 6d841bda..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_chemistry" -"description": "The following are multiple choice questions (with answers) about high\ - \ school chemistry.\n\n" -"group": "mmlu_alt_pv_01_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_computer_science.yaml deleted file mode 100644 index 9b7266fd..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_computer_science" -"description": "The following are multiple choice questions (with answers) about high\ - \ school computer science.\n\n" -"group": "mmlu_alt_pv_01_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_european_history.yaml deleted file mode 100644 index d19c3874..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_european_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_european_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school european history.\n\n" -"group": "mmlu_alt_pv_01_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_geography.yaml deleted file mode 100644 index a6693a49..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_geography.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_geography" -"description": "The following are multiple choice questions (with answers) about high\ - \ school geography.\n\n" -"group": "mmlu_alt_pv_01_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_high_school_geography" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_government_and_politics.yaml deleted file mode 100644 index 6f6644d5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_government_and_politics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_government_and_politics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school government and politics.\n\n" -"group": "mmlu_alt_pv_01_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_macroeconomics.yaml deleted file mode 100644 index dff29707..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_macroeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_macroeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school macroeconomics.\n\n" -"group": "mmlu_alt_pv_01_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_mathematics.yaml deleted file mode 100644 index 39ea0984..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_mathematics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school mathematics.\n\n" -"group": "mmlu_alt_pv_01_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_microeconomics.yaml deleted file mode 100644 index 327756b1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_microeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_microeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school microeconomics.\n\n" -"group": "mmlu_alt_pv_01_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_physics.yaml deleted file mode 100644 index 56c5999b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_physics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school physics.\n\n" -"group": "mmlu_alt_pv_01_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_high_school_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_psychology.yaml deleted file mode 100644 index 22c5c6ef..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_psychology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school psychology.\n\n" -"group": "mmlu_alt_pv_01_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_statistics.yaml deleted file mode 100644 index 4a7e279f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_statistics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_statistics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school statistics.\n\n" -"group": "mmlu_alt_pv_01_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_us_history.yaml deleted file mode 100644 index 223c548a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_us_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_us_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school us history.\n\n" -"group": "mmlu_alt_pv_01_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_world_history.yaml deleted file mode 100644 index ba3d32d8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_high_school_world_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_world_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school world history.\n\n" -"group": "mmlu_alt_pv_01_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_human_aging.yaml deleted file mode 100644 index 320dcb01..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_human_aging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_aging" -"description": "The following are multiple choice questions (with answers) about human\ - \ aging.\n\n" -"group": "mmlu_alt_pv_01_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_human_aging" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_human_sexuality.yaml deleted file mode 100644 index e21ee56f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_human_sexuality.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_sexuality" -"description": "The following are multiple choice questions (with answers) about human\ - \ sexuality.\n\n" -"group": "mmlu_alt_pv_01_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_human_sexuality" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_international_law.yaml deleted file mode 100644 index d1e4e13a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_international_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "international_law" -"description": "The following are multiple choice questions (with answers) about international\ - \ law.\n\n" -"group": "mmlu_alt_pv_01_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_international_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_jurisprudence.yaml deleted file mode 100644 index 3c28b447..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_jurisprudence.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "jurisprudence" -"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ - \n" -"group": "mmlu_alt_pv_01_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_jurisprudence" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_logical_fallacies.yaml deleted file mode 100644 index 36d1cb5b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_logical_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "logical_fallacies" -"description": "The following are multiple choice questions (with answers) about logical\ - \ fallacies.\n\n" -"group": "mmlu_alt_pv_01_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_machine_learning.yaml deleted file mode 100644 index a56ea1ba..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_machine_learning.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "machine_learning" -"description": "The following are multiple choice questions (with answers) about machine\ - \ learning.\n\n" -"group": "mmlu_alt_pv_01_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_machine_learning" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_management.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_management.yaml deleted file mode 100644 index fa362939..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_management.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "management" -"description": "The following are multiple choice questions (with answers) about management.\n\ - \n" -"group": "mmlu_alt_pv_01_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_management" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_marketing.yaml deleted file mode 100644 index 635afd3b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_marketing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "marketing" -"description": "The following are multiple choice questions (with answers) about marketing.\n\ - \n" -"group": "mmlu_alt_pv_01_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_marketing" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_medical_genetics.yaml deleted file mode 100644 index ac06e3df..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_medical_genetics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "medical_genetics" -"description": "The following are multiple choice questions (with answers) about medical\ - \ genetics.\n\n" -"group": "mmlu_alt_pv_01_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_medical_genetics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_miscellaneous.yaml deleted file mode 100644 index 9dfeadb3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_miscellaneous.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "miscellaneous" -"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ - \n" -"group": "mmlu_alt_pv_01_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_miscellaneous" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_moral_disputes.yaml deleted file mode 100644 index e74db2cb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_moral_disputes.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_disputes" -"description": "The following are multiple choice questions (with answers) about moral\ - \ disputes.\n\n" -"group": "mmlu_alt_pv_01_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_moral_disputes" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_moral_scenarios.yaml deleted file mode 100644 index 0eb99f43..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_moral_scenarios.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_scenarios" -"description": "The following are multiple choice questions (with answers) about moral\ - \ scenarios.\n\n" -"group": "mmlu_alt_pv_01_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_nutrition.yaml deleted file mode 100644 index a097ddbd..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_nutrition.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "nutrition" -"description": "The following are multiple choice questions (with answers) about nutrition.\n\ - \n" -"group": "mmlu_alt_pv_01_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_nutrition" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_philosophy.yaml deleted file mode 100644 index f0b463de..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_philosophy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "philosophy" -"description": "The following are multiple choice questions (with answers) about philosophy.\n\ - \n" -"group": "mmlu_alt_pv_01_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_philosophy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_prehistory.yaml deleted file mode 100644 index 588fbacb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_prehistory.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "prehistory" -"description": "The following are multiple choice questions (with answers) about prehistory.\n\ - \n" -"group": "mmlu_alt_pv_01_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_prehistory" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_professional_accounting.yaml deleted file mode 100644 index 5d58389b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_professional_accounting.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_accounting" -"description": "The following are multiple choice questions (with answers) about professional\ - \ accounting.\n\n" -"group": "mmlu_alt_pv_01_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_professional_accounting" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_professional_law.yaml deleted file mode 100644 index b7681cfd..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_professional_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_law" -"description": "The following are multiple choice questions (with answers) about professional\ - \ law.\n\n" -"group": "mmlu_alt_pv_01_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_professional_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_professional_medicine.yaml deleted file mode 100644 index 0d26d9d3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_professional_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_medicine" -"description": "The following are multiple choice questions (with answers) about professional\ - \ medicine.\n\n" -"group": "mmlu_alt_pv_01_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_professional_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_professional_psychology.yaml deleted file mode 100644 index 7a979cf0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_professional_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_psychology" -"description": "The following are multiple choice questions (with answers) about professional\ - \ psychology.\n\n" -"group": "mmlu_alt_pv_01_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_professional_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_public_relations.yaml deleted file mode 100644 index 87a8a946..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_public_relations.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "public_relations" -"description": "The following are multiple choice questions (with answers) about public\ - \ relations.\n\n" -"group": "mmlu_alt_pv_01_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_public_relations" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_security_studies.yaml deleted file mode 100644 index a89be56e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_security_studies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "security_studies" -"description": "The following are multiple choice questions (with answers) about security\ - \ studies.\n\n" -"group": "mmlu_alt_pv_01_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_security_studies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_sociology.yaml deleted file mode 100644 index 7e2bc065..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_sociology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sociology" -"description": "The following are multiple choice questions (with answers) about sociology.\n\ - \n" -"group": "mmlu_alt_pv_01_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_sociology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_us_foreign_policy.yaml deleted file mode 100644 index 02016eb6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_us_foreign_policy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "us_foreign_policy" -"description": "The following are multiple choice questions (with answers) about us\ - \ foreign policy.\n\n" -"group": "mmlu_alt_pv_01_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_virology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_virology.yaml deleted file mode 100644 index d08227a2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_virology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "virology" -"description": "The following are multiple choice questions (with answers) about virology.\n\ - \n" -"group": "mmlu_alt_pv_01_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_virology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_world_religions.yaml deleted file mode 100644 index 918db68b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_01/mmlu_world_religions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "world_religions" -"description": "The following are multiple choice questions (with answers) about world\ - \ religions.\n\n" -"group": "mmlu_alt_pv_01_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_01_world_religions" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/_default_template_yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/_default_template_yaml deleted file mode 100644 index 083fb599..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/_default_template_yaml +++ /dev/null @@ -1,13 +0,0 @@ -dataset_path: hails/mmlu_no_train # a copy of `cais/mmlu` with no auxiliary_train split -test_split: test -fewshot_split: dev -fewshot_config: - sampler: first_n -output_type: multiple_choice -doc_to_text: "Q: {{question.strip()}}\nA:" -doc_to_choice: choices -doc_to_target: answer -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/_mmlu.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/_mmlu.yaml deleted file mode 100644 index c0ec579b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/_mmlu.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: mmlu_alt_pv_02 -task: - - mmlu_alt_pv_02_stem - - mmlu_alt_pv_02_other - - mmlu_alt_pv_02_social_sciences - - mmlu_alt_pv_02_humanities diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_abstract_algebra.yaml deleted file mode 100644 index 8bd07995..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_abstract_algebra.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "abstract_algebra" -"description": "The following are multiple choice questions (with answers) about abstract\ - \ algebra.\n\n" -"group": "mmlu_alt_pv_02_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_anatomy.yaml deleted file mode 100644 index cdc0dbce..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_anatomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "anatomy" -"description": "The following are multiple choice questions (with answers) about anatomy.\n\ - \n" -"group": "mmlu_alt_pv_02_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_anatomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_astronomy.yaml deleted file mode 100644 index 484cbe37..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_astronomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "astronomy" -"description": "The following are multiple choice questions (with answers) about astronomy.\n\ - \n" -"group": "mmlu_alt_pv_02_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_astronomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_business_ethics.yaml deleted file mode 100644 index e626b34a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_business_ethics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "business_ethics" -"description": "The following are multiple choice questions (with answers) about business\ - \ ethics.\n\n" -"group": "mmlu_alt_pv_02_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_business_ethics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_clinical_knowledge.yaml deleted file mode 100644 index 547e3067..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_clinical_knowledge.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "clinical_knowledge" -"description": "The following are multiple choice questions (with answers) about clinical\ - \ knowledge.\n\n" -"group": "mmlu_alt_pv_02_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_college_biology.yaml deleted file mode 100644 index 2127089a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_college_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_biology" -"description": "The following are multiple choice questions (with answers) about college\ - \ biology.\n\n" -"group": "mmlu_alt_pv_02_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_college_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_college_chemistry.yaml deleted file mode 100644 index 75103c15..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_college_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_chemistry" -"description": "The following are multiple choice questions (with answers) about college\ - \ chemistry.\n\n" -"group": "mmlu_alt_pv_02_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_college_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_college_computer_science.yaml deleted file mode 100644 index 49e60548..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_college_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_computer_science" -"description": "The following are multiple choice questions (with answers) about college\ - \ computer science.\n\n" -"group": "mmlu_alt_pv_02_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_college_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_college_mathematics.yaml deleted file mode 100644 index c450b9a6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_college_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_mathematics" -"description": "The following are multiple choice questions (with answers) about college\ - \ mathematics.\n\n" -"group": "mmlu_alt_pv_02_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_college_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_college_medicine.yaml deleted file mode 100644 index 695e45be..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_college_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_medicine" -"description": "The following are multiple choice questions (with answers) about college\ - \ medicine.\n\n" -"group": "mmlu_alt_pv_02_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_college_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_college_physics.yaml deleted file mode 100644 index db409508..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_college_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_physics" -"description": "The following are multiple choice questions (with answers) about college\ - \ physics.\n\n" -"group": "mmlu_alt_pv_02_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_college_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_computer_security.yaml deleted file mode 100644 index 3bad6297..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_computer_security.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "computer_security" -"description": "The following are multiple choice questions (with answers) about computer\ - \ security.\n\n" -"group": "mmlu_alt_pv_02_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_computer_security" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_conceptual_physics.yaml deleted file mode 100644 index cfc363ae..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_conceptual_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "conceptual_physics" -"description": "The following are multiple choice questions (with answers) about conceptual\ - \ physics.\n\n" -"group": "mmlu_alt_pv_02_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_econometrics.yaml deleted file mode 100644 index 94aeee69..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_econometrics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "econometrics" -"description": "The following are multiple choice questions (with answers) about econometrics.\n\ - \n" -"group": "mmlu_alt_pv_02_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_econometrics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_electrical_engineering.yaml deleted file mode 100644 index 189fdc9d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_electrical_engineering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "electrical_engineering" -"description": "The following are multiple choice questions (with answers) about electrical\ - \ engineering.\n\n" -"group": "mmlu_alt_pv_02_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_elementary_mathematics.yaml deleted file mode 100644 index 9ebbb836..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_elementary_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "elementary_mathematics" -"description": "The following are multiple choice questions (with answers) about elementary\ - \ mathematics.\n\n" -"group": "mmlu_alt_pv_02_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_formal_logic.yaml deleted file mode 100644 index 7aa10a12..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_formal_logic.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_logic" -"description": "The following are multiple choice questions (with answers) about formal\ - \ logic.\n\n" -"group": "mmlu_alt_pv_02_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_formal_logic" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_global_facts.yaml deleted file mode 100644 index 180635a1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_global_facts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "global_facts" -"description": "The following are multiple choice questions (with answers) about global\ - \ facts.\n\n" -"group": "mmlu_alt_pv_02_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_global_facts" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_biology.yaml deleted file mode 100644 index 50ea4496..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_biology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school biology.\n\n" -"group": "mmlu_alt_pv_02_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_high_school_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_chemistry.yaml deleted file mode 100644 index 82279900..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_chemistry" -"description": "The following are multiple choice questions (with answers) about high\ - \ school chemistry.\n\n" -"group": "mmlu_alt_pv_02_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_computer_science.yaml deleted file mode 100644 index 5d48b507..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_computer_science" -"description": "The following are multiple choice questions (with answers) about high\ - \ school computer science.\n\n" -"group": "mmlu_alt_pv_02_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_european_history.yaml deleted file mode 100644 index 78c6ace6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_european_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_european_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school european history.\n\n" -"group": "mmlu_alt_pv_02_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_geography.yaml deleted file mode 100644 index bfc505c3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_geography.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_geography" -"description": "The following are multiple choice questions (with answers) about high\ - \ school geography.\n\n" -"group": "mmlu_alt_pv_02_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_high_school_geography" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_government_and_politics.yaml deleted file mode 100644 index 4581c806..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_government_and_politics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_government_and_politics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school government and politics.\n\n" -"group": "mmlu_alt_pv_02_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_macroeconomics.yaml deleted file mode 100644 index 0ac0d5e1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_macroeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_macroeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school macroeconomics.\n\n" -"group": "mmlu_alt_pv_02_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_mathematics.yaml deleted file mode 100644 index ddbd5349..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_mathematics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school mathematics.\n\n" -"group": "mmlu_alt_pv_02_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_microeconomics.yaml deleted file mode 100644 index 9dddda2f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_microeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_microeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school microeconomics.\n\n" -"group": "mmlu_alt_pv_02_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_physics.yaml deleted file mode 100644 index 3337914e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_physics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school physics.\n\n" -"group": "mmlu_alt_pv_02_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_high_school_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_psychology.yaml deleted file mode 100644 index b32de65e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_psychology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school psychology.\n\n" -"group": "mmlu_alt_pv_02_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_statistics.yaml deleted file mode 100644 index c9716f98..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_statistics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_statistics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school statistics.\n\n" -"group": "mmlu_alt_pv_02_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_us_history.yaml deleted file mode 100644 index 292b857e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_us_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_us_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school us history.\n\n" -"group": "mmlu_alt_pv_02_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_world_history.yaml deleted file mode 100644 index 2449fc17..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_high_school_world_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_world_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school world history.\n\n" -"group": "mmlu_alt_pv_02_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_human_aging.yaml deleted file mode 100644 index a7c912ef..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_human_aging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_aging" -"description": "The following are multiple choice questions (with answers) about human\ - \ aging.\n\n" -"group": "mmlu_alt_pv_02_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_human_aging" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_human_sexuality.yaml deleted file mode 100644 index 1cc26aae..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_human_sexuality.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_sexuality" -"description": "The following are multiple choice questions (with answers) about human\ - \ sexuality.\n\n" -"group": "mmlu_alt_pv_02_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_human_sexuality" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_international_law.yaml deleted file mode 100644 index 55859077..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_international_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "international_law" -"description": "The following are multiple choice questions (with answers) about international\ - \ law.\n\n" -"group": "mmlu_alt_pv_02_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_international_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_jurisprudence.yaml deleted file mode 100644 index 5273a53e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_jurisprudence.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "jurisprudence" -"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ - \n" -"group": "mmlu_alt_pv_02_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_jurisprudence" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_logical_fallacies.yaml deleted file mode 100644 index 822c67dc..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_logical_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "logical_fallacies" -"description": "The following are multiple choice questions (with answers) about logical\ - \ fallacies.\n\n" -"group": "mmlu_alt_pv_02_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_machine_learning.yaml deleted file mode 100644 index 0a3bb9d5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_machine_learning.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "machine_learning" -"description": "The following are multiple choice questions (with answers) about machine\ - \ learning.\n\n" -"group": "mmlu_alt_pv_02_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_machine_learning" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_management.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_management.yaml deleted file mode 100644 index e4af9ba0..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_management.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "management" -"description": "The following are multiple choice questions (with answers) about management.\n\ - \n" -"group": "mmlu_alt_pv_02_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_management" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_marketing.yaml deleted file mode 100644 index 1a23ba70..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_marketing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "marketing" -"description": "The following are multiple choice questions (with answers) about marketing.\n\ - \n" -"group": "mmlu_alt_pv_02_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_marketing" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_medical_genetics.yaml deleted file mode 100644 index c4792672..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_medical_genetics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "medical_genetics" -"description": "The following are multiple choice questions (with answers) about medical\ - \ genetics.\n\n" -"group": "mmlu_alt_pv_02_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_medical_genetics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_miscellaneous.yaml deleted file mode 100644 index f37e75ce..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_miscellaneous.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "miscellaneous" -"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ - \n" -"group": "mmlu_alt_pv_02_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_miscellaneous" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_moral_disputes.yaml deleted file mode 100644 index 0657ff03..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_moral_disputes.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_disputes" -"description": "The following are multiple choice questions (with answers) about moral\ - \ disputes.\n\n" -"group": "mmlu_alt_pv_02_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_moral_disputes" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_moral_scenarios.yaml deleted file mode 100644 index 4774d817..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_moral_scenarios.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_scenarios" -"description": "The following are multiple choice questions (with answers) about moral\ - \ scenarios.\n\n" -"group": "mmlu_alt_pv_02_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_nutrition.yaml deleted file mode 100644 index ccc9c922..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_nutrition.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "nutrition" -"description": "The following are multiple choice questions (with answers) about nutrition.\n\ - \n" -"group": "mmlu_alt_pv_02_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_nutrition" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_philosophy.yaml deleted file mode 100644 index 2daae4ee..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_philosophy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "philosophy" -"description": "The following are multiple choice questions (with answers) about philosophy.\n\ - \n" -"group": "mmlu_alt_pv_02_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_philosophy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_prehistory.yaml deleted file mode 100644 index ba217bbe..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_prehistory.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "prehistory" -"description": "The following are multiple choice questions (with answers) about prehistory.\n\ - \n" -"group": "mmlu_alt_pv_02_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_prehistory" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_professional_accounting.yaml deleted file mode 100644 index b9cbc5f7..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_professional_accounting.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_accounting" -"description": "The following are multiple choice questions (with answers) about professional\ - \ accounting.\n\n" -"group": "mmlu_alt_pv_02_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_professional_accounting" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_professional_law.yaml deleted file mode 100644 index 15828b99..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_professional_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_law" -"description": "The following are multiple choice questions (with answers) about professional\ - \ law.\n\n" -"group": "mmlu_alt_pv_02_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_professional_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_professional_medicine.yaml deleted file mode 100644 index a6800e9b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_professional_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_medicine" -"description": "The following are multiple choice questions (with answers) about professional\ - \ medicine.\n\n" -"group": "mmlu_alt_pv_02_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_professional_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_professional_psychology.yaml deleted file mode 100644 index 3dc609c2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_professional_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_psychology" -"description": "The following are multiple choice questions (with answers) about professional\ - \ psychology.\n\n" -"group": "mmlu_alt_pv_02_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_professional_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_public_relations.yaml deleted file mode 100644 index 3c798918..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_public_relations.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "public_relations" -"description": "The following are multiple choice questions (with answers) about public\ - \ relations.\n\n" -"group": "mmlu_alt_pv_02_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_public_relations" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_security_studies.yaml deleted file mode 100644 index 1b5086bb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_security_studies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "security_studies" -"description": "The following are multiple choice questions (with answers) about security\ - \ studies.\n\n" -"group": "mmlu_alt_pv_02_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_security_studies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_sociology.yaml deleted file mode 100644 index 726bf086..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_sociology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sociology" -"description": "The following are multiple choice questions (with answers) about sociology.\n\ - \n" -"group": "mmlu_alt_pv_02_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_sociology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_us_foreign_policy.yaml deleted file mode 100644 index 80962e7e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_us_foreign_policy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "us_foreign_policy" -"description": "The following are multiple choice questions (with answers) about us\ - \ foreign policy.\n\n" -"group": "mmlu_alt_pv_02_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_virology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_virology.yaml deleted file mode 100644 index 66cd00c4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_virology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "virology" -"description": "The following are multiple choice questions (with answers) about virology.\n\ - \n" -"group": "mmlu_alt_pv_02_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_virology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_world_religions.yaml deleted file mode 100644 index 92dd159a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_02/mmlu_world_religions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "world_religions" -"description": "The following are multiple choice questions (with answers) about world\ - \ religions.\n\n" -"group": "mmlu_alt_pv_02_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_02_world_religions" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/_default_template_yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/_default_template_yaml deleted file mode 100644 index 07f98304..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/_default_template_yaml +++ /dev/null @@ -1,13 +0,0 @@ -dataset_path: hails/mmlu_no_train # a copy of `cais/mmlu` with no auxiliary_train split -test_split: test -fewshot_split: dev -fewshot_config: - sampler: first_n -output_type: multiple_choice -doc_to_text: "Question: {{question.strip()}}\nAnswer:" -doc_to_choice: choices -doc_to_target: answer -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/_mmlu.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/_mmlu.yaml deleted file mode 100644 index 39e434a5..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/_mmlu.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: mmlu_alt_pv_03 -task: - - mmlu_alt_pv_03_stem - - mmlu_alt_pv_03_other - - mmlu_alt_pv_03_social_sciences - - mmlu_alt_pv_03_humanities diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_abstract_algebra.yaml deleted file mode 100644 index 7941d4a8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_abstract_algebra.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "abstract_algebra" -"description": "The following are multiple choice questions (with answers) about abstract\ - \ algebra.\n\n" -"group": "mmlu_alt_pv_03_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_anatomy.yaml deleted file mode 100644 index 6d04c321..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_anatomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "anatomy" -"description": "The following are multiple choice questions (with answers) about anatomy.\n\ - \n" -"group": "mmlu_alt_pv_03_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_anatomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_astronomy.yaml deleted file mode 100644 index d31997eb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_astronomy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "astronomy" -"description": "The following are multiple choice questions (with answers) about astronomy.\n\ - \n" -"group": "mmlu_alt_pv_03_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_astronomy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_business_ethics.yaml deleted file mode 100644 index aea729b4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_business_ethics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "business_ethics" -"description": "The following are multiple choice questions (with answers) about business\ - \ ethics.\n\n" -"group": "mmlu_alt_pv_03_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_business_ethics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_clinical_knowledge.yaml deleted file mode 100644 index 949c5fe4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_clinical_knowledge.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "clinical_knowledge" -"description": "The following are multiple choice questions (with answers) about clinical\ - \ knowledge.\n\n" -"group": "mmlu_alt_pv_03_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_college_biology.yaml deleted file mode 100644 index f2a947f2..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_college_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_biology" -"description": "The following are multiple choice questions (with answers) about college\ - \ biology.\n\n" -"group": "mmlu_alt_pv_03_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_college_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_college_chemistry.yaml deleted file mode 100644 index 29e01ad4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_college_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_chemistry" -"description": "The following are multiple choice questions (with answers) about college\ - \ chemistry.\n\n" -"group": "mmlu_alt_pv_03_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_college_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_college_computer_science.yaml deleted file mode 100644 index ae23319a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_college_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_computer_science" -"description": "The following are multiple choice questions (with answers) about college\ - \ computer science.\n\n" -"group": "mmlu_alt_pv_03_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_college_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_college_mathematics.yaml deleted file mode 100644 index b349a40a..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_college_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_mathematics" -"description": "The following are multiple choice questions (with answers) about college\ - \ mathematics.\n\n" -"group": "mmlu_alt_pv_03_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_college_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_college_medicine.yaml deleted file mode 100644 index ccc5fd2b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_college_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_medicine" -"description": "The following are multiple choice questions (with answers) about college\ - \ medicine.\n\n" -"group": "mmlu_alt_pv_03_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_college_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_college_physics.yaml deleted file mode 100644 index cdb18383..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_college_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "college_physics" -"description": "The following are multiple choice questions (with answers) about college\ - \ physics.\n\n" -"group": "mmlu_alt_pv_03_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_college_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_computer_security.yaml deleted file mode 100644 index b2ef889d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_computer_security.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "computer_security" -"description": "The following are multiple choice questions (with answers) about computer\ - \ security.\n\n" -"group": "mmlu_alt_pv_03_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_computer_security" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_conceptual_physics.yaml deleted file mode 100644 index 21517124..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_conceptual_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "conceptual_physics" -"description": "The following are multiple choice questions (with answers) about conceptual\ - \ physics.\n\n" -"group": "mmlu_alt_pv_03_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_econometrics.yaml deleted file mode 100644 index c42e5ca8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_econometrics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "econometrics" -"description": "The following are multiple choice questions (with answers) about econometrics.\n\ - \n" -"group": "mmlu_alt_pv_03_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_econometrics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_electrical_engineering.yaml deleted file mode 100644 index b3c61ff8..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_electrical_engineering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "electrical_engineering" -"description": "The following are multiple choice questions (with answers) about electrical\ - \ engineering.\n\n" -"group": "mmlu_alt_pv_03_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_elementary_mathematics.yaml deleted file mode 100644 index b533bbc1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_elementary_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "elementary_mathematics" -"description": "The following are multiple choice questions (with answers) about elementary\ - \ mathematics.\n\n" -"group": "mmlu_alt_pv_03_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_formal_logic.yaml deleted file mode 100644 index 02e4314b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_formal_logic.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "formal_logic" -"description": "The following are multiple choice questions (with answers) about formal\ - \ logic.\n\n" -"group": "mmlu_alt_pv_03_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_formal_logic" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_global_facts.yaml deleted file mode 100644 index 508ddfee..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_global_facts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "global_facts" -"description": "The following are multiple choice questions (with answers) about global\ - \ facts.\n\n" -"group": "mmlu_alt_pv_03_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_global_facts" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_biology.yaml deleted file mode 100644 index d12e03e3..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_biology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_biology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school biology.\n\n" -"group": "mmlu_alt_pv_03_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_high_school_biology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_chemistry.yaml deleted file mode 100644 index a7d6d032..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_chemistry.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_chemistry" -"description": "The following are multiple choice questions (with answers) about high\ - \ school chemistry.\n\n" -"group": "mmlu_alt_pv_03_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_computer_science.yaml deleted file mode 100644 index d909a678..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_computer_science.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_computer_science" -"description": "The following are multiple choice questions (with answers) about high\ - \ school computer science.\n\n" -"group": "mmlu_alt_pv_03_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_european_history.yaml deleted file mode 100644 index 03c4df82..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_european_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_european_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school european history.\n\n" -"group": "mmlu_alt_pv_03_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_geography.yaml deleted file mode 100644 index 1231d693..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_geography.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_geography" -"description": "The following are multiple choice questions (with answers) about high\ - \ school geography.\n\n" -"group": "mmlu_alt_pv_03_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_high_school_geography" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_government_and_politics.yaml deleted file mode 100644 index 2bb26aff..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_government_and_politics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_government_and_politics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school government and politics.\n\n" -"group": "mmlu_alt_pv_03_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_macroeconomics.yaml deleted file mode 100644 index 62db2eaf..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_macroeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_macroeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school macroeconomics.\n\n" -"group": "mmlu_alt_pv_03_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_mathematics.yaml deleted file mode 100644 index 8a690a27..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_mathematics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_mathematics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school mathematics.\n\n" -"group": "mmlu_alt_pv_03_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_microeconomics.yaml deleted file mode 100644 index e924615d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_microeconomics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_microeconomics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school microeconomics.\n\n" -"group": "mmlu_alt_pv_03_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_physics.yaml deleted file mode 100644 index 095d9964..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_physics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_physics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school physics.\n\n" -"group": "mmlu_alt_pv_03_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_high_school_physics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_psychology.yaml deleted file mode 100644 index a0957767..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_psychology" -"description": "The following are multiple choice questions (with answers) about high\ - \ school psychology.\n\n" -"group": "mmlu_alt_pv_03_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_statistics.yaml deleted file mode 100644 index 2a7ddd5b..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_statistics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_statistics" -"description": "The following are multiple choice questions (with answers) about high\ - \ school statistics.\n\n" -"group": "mmlu_alt_pv_03_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_us_history.yaml deleted file mode 100644 index 7c6e9d12..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_us_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_us_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school us history.\n\n" -"group": "mmlu_alt_pv_03_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_world_history.yaml deleted file mode 100644 index 77107f32..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_high_school_world_history.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "high_school_world_history" -"description": "The following are multiple choice questions (with answers) about high\ - \ school world history.\n\n" -"group": "mmlu_alt_pv_03_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_human_aging.yaml deleted file mode 100644 index 9aeaece1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_human_aging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_aging" -"description": "The following are multiple choice questions (with answers) about human\ - \ aging.\n\n" -"group": "mmlu_alt_pv_03_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_human_aging" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_human_sexuality.yaml deleted file mode 100644 index 0dd688bf..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_human_sexuality.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "human_sexuality" -"description": "The following are multiple choice questions (with answers) about human\ - \ sexuality.\n\n" -"group": "mmlu_alt_pv_03_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_human_sexuality" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_international_law.yaml deleted file mode 100644 index d284aa6c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_international_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "international_law" -"description": "The following are multiple choice questions (with answers) about international\ - \ law.\n\n" -"group": "mmlu_alt_pv_03_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_international_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_jurisprudence.yaml deleted file mode 100644 index 43ecb1c6..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_jurisprudence.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "jurisprudence" -"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ - \n" -"group": "mmlu_alt_pv_03_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_jurisprudence" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_logical_fallacies.yaml deleted file mode 100644 index 89dcf27d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_logical_fallacies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "logical_fallacies" -"description": "The following are multiple choice questions (with answers) about logical\ - \ fallacies.\n\n" -"group": "mmlu_alt_pv_03_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_machine_learning.yaml deleted file mode 100644 index 1131165e..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_machine_learning.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "machine_learning" -"description": "The following are multiple choice questions (with answers) about machine\ - \ learning.\n\n" -"group": "mmlu_alt_pv_03_stem" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_machine_learning" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_management.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_management.yaml deleted file mode 100644 index 72400f1f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_management.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "management" -"description": "The following are multiple choice questions (with answers) about management.\n\ - \n" -"group": "mmlu_alt_pv_03_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_management" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_marketing.yaml deleted file mode 100644 index 52332eed..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_marketing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "marketing" -"description": "The following are multiple choice questions (with answers) about marketing.\n\ - \n" -"group": "mmlu_alt_pv_03_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_marketing" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_medical_genetics.yaml deleted file mode 100644 index 8efa7e9c..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_medical_genetics.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "medical_genetics" -"description": "The following are multiple choice questions (with answers) about medical\ - \ genetics.\n\n" -"group": "mmlu_alt_pv_03_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_medical_genetics" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_miscellaneous.yaml deleted file mode 100644 index 39b9ae77..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_miscellaneous.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "miscellaneous" -"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ - \n" -"group": "mmlu_alt_pv_03_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_miscellaneous" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_moral_disputes.yaml deleted file mode 100644 index eb08c777..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_moral_disputes.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_disputes" -"description": "The following are multiple choice questions (with answers) about moral\ - \ disputes.\n\n" -"group": "mmlu_alt_pv_03_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_moral_disputes" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_moral_scenarios.yaml deleted file mode 100644 index 84cc1692..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_moral_scenarios.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "moral_scenarios" -"description": "The following are multiple choice questions (with answers) about moral\ - \ scenarios.\n\n" -"group": "mmlu_alt_pv_03_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_nutrition.yaml deleted file mode 100644 index c5f4a05f..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_nutrition.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "nutrition" -"description": "The following are multiple choice questions (with answers) about nutrition.\n\ - \n" -"group": "mmlu_alt_pv_03_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_nutrition" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_philosophy.yaml deleted file mode 100644 index a7c3408d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_philosophy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "philosophy" -"description": "The following are multiple choice questions (with answers) about philosophy.\n\ - \n" -"group": "mmlu_alt_pv_03_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_philosophy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_prehistory.yaml deleted file mode 100644 index 20294008..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_prehistory.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "prehistory" -"description": "The following are multiple choice questions (with answers) about prehistory.\n\ - \n" -"group": "mmlu_alt_pv_03_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_prehistory" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_professional_accounting.yaml deleted file mode 100644 index 13f6cefb..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_professional_accounting.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_accounting" -"description": "The following are multiple choice questions (with answers) about professional\ - \ accounting.\n\n" -"group": "mmlu_alt_pv_03_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_professional_accounting" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_professional_law.yaml deleted file mode 100644 index 461f5ba1..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_professional_law.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_law" -"description": "The following are multiple choice questions (with answers) about professional\ - \ law.\n\n" -"group": "mmlu_alt_pv_03_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_professional_law" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_professional_medicine.yaml deleted file mode 100644 index f0a90066..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_professional_medicine.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_medicine" -"description": "The following are multiple choice questions (with answers) about professional\ - \ medicine.\n\n" -"group": "mmlu_alt_pv_03_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_professional_medicine" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_professional_psychology.yaml deleted file mode 100644 index f61d3e40..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_professional_psychology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "professional_psychology" -"description": "The following are multiple choice questions (with answers) about professional\ - \ psychology.\n\n" -"group": "mmlu_alt_pv_03_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_professional_psychology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_public_relations.yaml deleted file mode 100644 index 2b2e2a97..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_public_relations.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "public_relations" -"description": "The following are multiple choice questions (with answers) about public\ - \ relations.\n\n" -"group": "mmlu_alt_pv_03_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_public_relations" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_security_studies.yaml deleted file mode 100644 index 3e8896d4..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_security_studies.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "security_studies" -"description": "The following are multiple choice questions (with answers) about security\ - \ studies.\n\n" -"group": "mmlu_alt_pv_03_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_security_studies" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_sociology.yaml deleted file mode 100644 index d808d4ec..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_sociology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "sociology" -"description": "The following are multiple choice questions (with answers) about sociology.\n\ - \n" -"group": "mmlu_alt_pv_03_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_sociology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_us_foreign_policy.yaml deleted file mode 100644 index f3c68319..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_us_foreign_policy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "us_foreign_policy" -"description": "The following are multiple choice questions (with answers) about us\ - \ foreign policy.\n\n" -"group": "mmlu_alt_pv_03_social_sciences" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_virology.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_virology.yaml deleted file mode 100644 index d710576d..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_virology.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "virology" -"description": "The following are multiple choice questions (with answers) about virology.\n\ - \n" -"group": "mmlu_alt_pv_03_other" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_virology" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_world_religions.yaml deleted file mode 100644 index e8367f96..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/prompt_variation/style_03/mmlu_world_religions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -"dataset_name": "world_religions" -"description": "The following are multiple choice questions (with answers) about world\ - \ religions.\n\n" -"group": "mmlu_alt_pv_03_humanities" -"include": "_default_template_yaml" -"task": "mmlu_alt_pv_03_world_religions" diff --git a/lm_eval/tasks/mmlu/alternative_worlds/styles.py b/lm_eval/tasks/mmlu/alternative_worlds/styles.py deleted file mode 100644 index 24656672..00000000 --- a/lm_eval/tasks/mmlu/alternative_worlds/styles.py +++ /dev/null @@ -1,114 +0,0 @@ -import string -from functools import partial - - -def doc_to_text_base(alphabet, style, doc): - - choices = doc["choices"] - num = len(choices) - - letter_list = [style.format(letter) for letter in alphabet[0:num]] - - if "\t" in style: - choice_string = "{}{}" - else: - choice_string = "{} {}" - - doc_to_text = "\n".join( - [ - "Question: " + doc["question"].strip() - ] - + [" ".join(choice_string.format(i, j) for i, j in zip(letter_list, choices))] - + ["Answer:"] - ) - - return doc_to_text - - -# Full continuation -def choice_A(doc): - return doc["choices"] - -def answer_A(doc): - return doc["choices"][doc["answer"]] - -# Letters only -def choice_B(alphabet, style, doc): - - choices = doc["choices"] - num = len(choices) - - letter_list = [style.format(letter) for letter in alphabet[0:num]] - if "\t" in style: - letter_list = [letter.replace("\t", "") for letter in letter_list] - - return letter_list - -def answer_B(alphabet, style, doc): - - choices = doc["choices"] - num = len(choices) - - letter_list = [style.format(letter) for letter in alphabet[0:num]] - if "\t" in style: - letter_list = [letter.replace("\t", "") for letter in letter_list] - - return letter_list[doc["answer"]] - -# Letters + Full continuation -def choice_C(alphabet, style, doc): - - choices = doc["choices"] - num = len(choices) - - letter_list = [style.format(letter) for letter in alphabet[0:num]] - if "\t" not in style: - letter_list = [letter + " " for letter in letter_list] - - return [letter + choice for letter, choice in zip(letter_list, choices)] - -def answer_C(alphabet, style, doc): - - choices = doc["choices"] - num = len(choices) - - letter_list = [style.format(letter) for letter in alphabet[0:num]] - if "\t" not in style: - letter_list = [letter + " " for letter in letter_list] - - return [letter + choice for letter, choice in zip(letter_list, choices)][doc["answer"]] - - -template_01 = partial(doc_to_text_base, string.ascii_lowercase, "({})") -choice_01a = choice_A -choice_01b = partial(choice_B, string.ascii_lowercase, "({})") -choice_01c = partial(choice_C, string.ascii_lowercase, "({})") -template_02 = partial(doc_to_text_base, string.ascii_lowercase, "{})") -choice_02a = choice_A -choice_02b = partial(choice_B, string.ascii_lowercase, "{})") -choice_02c = partial(choice_C, string.ascii_lowercase, "{})") -template_03 = partial(doc_to_text_base, string.ascii_lowercase, "{}.") -choice_03a = choice_A -choice_03b = partial(choice_B, string.ascii_lowercase, "{}.") -choice_03c = partial(choice_C, string.ascii_lowercase, "{}.") -template_04 = partial(doc_to_text_base, string.ascii_lowercase, "{}\t") -choice_04a = choice_A -choice_04b = partial(choice_B, string.ascii_lowercase, "{}\t") -choice_04c = partial(choice_C, string.ascii_lowercase, "{}\t") -template_05 = partial(doc_to_text_base, string.ascii_uppercase, "({})") -target_05a = answer_A -choice_05a = choice_A -choice_05b = partial(choice_B, string.ascii_uppercase, "({})") -choice_05c = partial(choice_C, string.ascii_uppercase, "({})") -template_06 = partial(doc_to_text_base, string.ascii_uppercase, "{})") -choice_06a = choice_A -choice_06b = partial(choice_B, string.ascii_uppercase, "{})") -choice_06c = partial(choice_C, string.ascii_uppercase, "{})") -template_07 = partial(doc_to_text_base, string.ascii_uppercase, "{}.") -choice_07a = choice_A -choice_07b = partial(choice_B, string.ascii_uppercase, "{}.") -choice_07c = partial(choice_C, string.ascii_uppercase, "{}.") -template_08 = partial(doc_to_text_base, string.ascii_uppercase, "{}\t") -choice_08a = choice_A -choice_08b = partial(choice_B, string.ascii_uppercase, "{}\t") -choice_08c = partial(choice_C, string.ascii_uppercase, "{}\t") diff --git a/lm_eval/tasks/piqa/alternative_worlds/README.md b/lm_eval/tasks/piqa/alternative_worlds/README.md deleted file mode 100644 index 93600ae1..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/README.md +++ /dev/null @@ -1,20 +0,0 @@ - - -Investigate affect of letter options -- (A) -- A) -- A. -- A\t -- (a) -- a) -- a. -- a\t - -Answer types: -- letters only - - original option - - just letter -- letters + continuation - - original option - - just letter -- continuation diff --git a/lm_eval/tasks/piqa/alternative_worlds/output_variation/_piqa_alt_ov_yaml b/lm_eval/tasks/piqa/alternative_worlds/output_variation/_piqa_alt_ov_yaml deleted file mode 100644 index d26d2ae3..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/output_variation/_piqa_alt_ov_yaml +++ /dev/null @@ -1,20 +0,0 @@ -dataset_path: piqa -output_type: multiple_choice -training_split: train -validation_split: validation -doc_to_text: "Question: {{goal}}\nAnswer:" -doc_to_target: label -doc_to_choice: "{{[sol1, sol2]}}" -should_decontaminate: true -metric_list: - - metric: acc - aggregation: mean - higher_is_better: true - - metric: acc_norm - aggregation: mean - higher_is_better: true - - metric: brier_score - aggregation: brier_score - higher_is_better: false -metadata: - - version: 1.0 diff --git a/lm_eval/tasks/piqa/alternative_worlds/output_variation/piqa_alt_ov.yaml b/lm_eval/tasks/piqa/alternative_worlds/output_variation/piqa_alt_ov.yaml deleted file mode 100644 index da917426..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/output_variation/piqa_alt_ov.yaml +++ /dev/null @@ -1,10 +0,0 @@ -group: piqa_alt_ov -task: - - piqa_alt_ov_01 - - piqa_alt_ov_02 - - piqa_alt_ov_03 - - piqa_alt_ov_04 - - piqa_alt_ov_05 - - piqa_alt_ov_06 - - piqa_alt_ov_07 - - piqa_alt_ov_08 diff --git a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_01/a.yaml b/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_01/a.yaml deleted file mode 100644 index a19793a7..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_01/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_piqa_alt_ov_yaml -group: piqa_alt_ov_01 -task: piqa_alt_ov_01a -doc_to_text: !function ../styles.template_01 -doc_to_choice: !function ../styles.choice_01a -doc_to_decontamination_query: !function ../styles.template_01 diff --git a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_01/b.yaml b/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_01/b.yaml deleted file mode 100644 index 6ae4b304..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_01/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_piqa_alt_ov_yaml -group: piqa_alt_ov_01 -task: piqa_alt_ov_01b -doc_to_text: !function ../styles.template_01 -doc_to_choice: !function ../styles.choice_01b -doc_to_decontamination_query: !function ../styles.template_01 diff --git a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_01/c.yaml b/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_01/c.yaml deleted file mode 100644 index 84cbd654..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_01/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_piqa_alt_ov_yaml -group: piqa_alt_ov_01 -task: piqa_alt_ov_01c -doc_to_text: !function ../styles.template_01 -doc_to_choice: !function ../styles.choice_01c -doc_to_decontamination_query: !function ../styles.template_01 diff --git a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_02/a.yaml b/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_02/a.yaml deleted file mode 100644 index ad48ca78..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_02/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_piqa_alt_ov_yaml -group: piqa_alt_ov_02 -task: piqa_alt_ov_02a -doc_to_text: !function ../styles.template_02 -doc_to_choice: !function ../styles.choice_02a -doc_to_decontamination_query: !function ../styles.template_02 diff --git a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_02/b.yaml b/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_02/b.yaml deleted file mode 100644 index a7c2b84a..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_02/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_piqa_alt_ov_yaml -group: piqa_alt_ov_02 -task: piqa_alt_ov_02b -doc_to_text: !function ../styles.template_02 -doc_to_choice: !function ../styles.choice_02b -doc_to_decontamination_query: !function ../styles.template_02 diff --git a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_02/c.yaml b/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_02/c.yaml deleted file mode 100644 index e2f45ff9..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_02/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_piqa_alt_ov_yaml -group: piqa_alt_ov_02 -task: piqa_alt_ov_02c -doc_to_text: !function ../styles.template_02 -doc_to_choice: !function ../styles.choice_02c -doc_to_decontamination_query: !function ../styles.template_02 diff --git a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_03/a.yaml b/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_03/a.yaml deleted file mode 100644 index 70a5ec8c..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_03/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_piqa_alt_ov_yaml -group: piqa_alt_ov_03 -task: piqa_alt_ov_03a -doc_to_text: !function ../styles.template_03 -doc_to_choice: !function ../styles.choice_03a -doc_to_decontamination_query: !function ../styles.template_03 diff --git a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_03/b.yaml b/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_03/b.yaml deleted file mode 100644 index af73914b..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_03/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_piqa_alt_ov_yaml -group: piqa_alt_ov_03 -task: piqa_alt_ov_03b -doc_to_text: !function ../styles.template_03 -doc_to_choice: !function ../styles.choice_03b -doc_to_decontamination_query: !function ../styles.template_03 diff --git a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_03/c.yaml b/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_03/c.yaml deleted file mode 100644 index 3484705f..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_03/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_piqa_alt_ov_yaml -group: piqa_alt_ov_03 -task: piqa_alt_ov_03c -doc_to_text: !function ../styles.template_03 -doc_to_choice: !function ../styles.choice_03c -doc_to_decontamination_query: !function ../styles.template_03 diff --git a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_04/a.yaml b/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_04/a.yaml deleted file mode 100644 index 79a36448..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_04/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_piqa_alt_ov_yaml -group: piqa_alt_ov_04 -task: piqa_alt_ov_04a -doc_to_text: !function ../styles.template_04 -doc_to_choice: !function ../styles.choice_04a -doc_to_decontamination_query: !function ../styles.template_04 diff --git a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_04/b.yaml b/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_04/b.yaml deleted file mode 100644 index 36b3347c..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_04/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_piqa_alt_ov_yaml -group: piqa_alt_ov_04 -task: piqa_alt_ov_04b -doc_to_text: !function ../styles.template_04 -doc_to_choice: !function ../styles.choice_04b -doc_to_decontamination_query: !function ../styles.template_04 diff --git a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_04/c.yaml b/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_04/c.yaml deleted file mode 100644 index 3e62b456..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_04/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_piqa_alt_ov_yaml -group: piqa_alt_ov_04 -task: piqa_alt_ov_04c -doc_to_text: !function ../styles.template_04 -doc_to_choice: !function ../styles.choice_04c -doc_to_decontamination_query: !function ../styles.template_04 diff --git a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_05/a.yaml b/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_05/a.yaml deleted file mode 100644 index 54c4a980..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_05/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_piqa_alt_ov_yaml -group: piqa_alt_ov_05 -task: piqa_alt_ov_05a -doc_to_text: !function ../styles.template_05 -doc_to_choice: !function ../styles.choice_05a -doc_to_decontamination_query: !function ../styles.template_05 diff --git a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_05/b.yaml b/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_05/b.yaml deleted file mode 100644 index 2954cc53..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_05/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_piqa_alt_ov_yaml -group: piqa_alt_ov_05 -task: piqa_alt_ov_05b -doc_to_text: !function ../styles.template_05 -doc_to_choice: !function ../styles.choice_05b -doc_to_decontamination_query: !function ../styles.template_05 diff --git a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_05/c.yaml b/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_05/c.yaml deleted file mode 100644 index b604e1d2..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_05/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_piqa_alt_ov_yaml -group: piqa_alt_ov_05 -task: piqa_alt_ov_05c -doc_to_text: !function ../styles.template_05 -doc_to_choice: !function ../styles.choice_05c -doc_to_decontamination_query: !function ../styles.template_05 diff --git a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_06/a.yaml b/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_06/a.yaml deleted file mode 100644 index d0bc222d..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_06/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_piqa_alt_ov_yaml -group: piqa_alt_ov_06 -task: piqa_alt_ov_06a -doc_to_text: !function ../styles.template_06 -doc_to_choice: !function ../styles.choice_06a -doc_to_decontamination_query: !function ../styles.template_06 diff --git a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_06/b.yaml b/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_06/b.yaml deleted file mode 100644 index 393db4a0..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_06/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_piqa_alt_ov_yaml -group: piqa_alt_ov_06 -task: piqa_alt_ov_06b -doc_to_text: !function ../styles.template_06 -doc_to_choice: !function ../styles.choice_06b -doc_to_decontamination_query: !function ../styles.template_06 diff --git a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_06/c.yaml b/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_06/c.yaml deleted file mode 100644 index 2be7bf0d..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_06/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_piqa_alt_ov_yaml -group: piqa_alt_ov_06 -task: piqa_alt_ov_06c -doc_to_text: !function ../styles.template_06 -doc_to_choice: !function ../styles.choice_06c -doc_to_decontamination_query: !function ../styles.template_06 diff --git a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_07/a.yaml b/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_07/a.yaml deleted file mode 100644 index 4a95f9e8..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_07/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_piqa_alt_ov_yaml -group: piqa_alt_ov_07 -task: piqa_alt_ov_07a -doc_to_text: !function ../styles.template_07 -doc_to_choice: !function ../styles.choice_07a -doc_to_decontamination_query: !function ../styles.template_07 diff --git a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_07/b.yaml b/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_07/b.yaml deleted file mode 100644 index b6bc0432..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_07/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_piqa_alt_ov_yaml -group: piqa_alt_ov_07 -task: piqa_alt_ov_07b -doc_to_text: !function ../styles.template_07 -doc_to_choice: !function ../styles.choice_07b -doc_to_decontamination_query: !function ../styles.template_07 diff --git a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_07/c.yaml b/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_07/c.yaml deleted file mode 100644 index 113fb52b..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_07/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_piqa_alt_ov_yaml -group: piqa_alt_ov_07 -task: piqa_alt_ov_07c -doc_to_text: !function ../styles.template_07 -doc_to_choice: !function ../styles.choice_07c -doc_to_decontamination_query: !function ../styles.template_07 diff --git a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_08/a.yaml b/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_08/a.yaml deleted file mode 100644 index 7f56c485..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_08/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_piqa_alt_ov_yaml -group: piqa_alt_ov_08 -task: piqa_alt_ov_08a -doc_to_text: !function ../styles.template_08 -doc_to_choice: !function ../styles.choice_08a -doc_to_decontamination_query: !function ../styles.template_08 diff --git a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_08/b.yaml b/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_08/b.yaml deleted file mode 100644 index 4458cd50..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_08/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_piqa_alt_ov_yaml -group: piqa_alt_ov_08 -task: piqa_alt_ov_08b -doc_to_text: !function ../styles.template_08 -doc_to_choice: !function ../styles.choice_08b -doc_to_decontamination_query: !function ../styles.template_08 diff --git a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_08/c.yaml b/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_08/c.yaml deleted file mode 100644 index 01e77487..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/output_variation/style_08/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_piqa_alt_ov_yaml -group: piqa_alt_ov_08 -task: piqa_alt_ov_08c -doc_to_text: !function ../styles.template_08 -doc_to_choice: !function ../styles.choice_08c -doc_to_decontamination_query: !function ../styles.template_08 diff --git a/lm_eval/tasks/piqa/alternative_worlds/output_variation/styles.py b/lm_eval/tasks/piqa/alternative_worlds/output_variation/styles.py deleted file mode 100644 index 68447e2d..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/output_variation/styles.py +++ /dev/null @@ -1,90 +0,0 @@ -import string -from functools import partial - - -def doc_to_text_base(alphabet, style, doc): - - choices = [doc["sol1"], doc["sol2"]] - num = len(choices) - - letter_list = [style.format(letter) for letter in alphabet[0:num]] - - if "\t" in style: - choice_string = "{}{}" - else: - choice_string = "{} {}" - - doc_to_text = "\n".join( - [ - "Question: " + doc["goal"], - ] - + [choice_string.format(i, j) for i, j in zip(letter_list, choices)] - + ["Answer:"] - ) - - return doc_to_text - - -# Full continuation -def choice_A(doc): - return [doc["sol1"], doc["sol2"]] - - -# Letters only -def choice_B(alphabet, style, doc): - - choices = [doc["sol1"], doc["sol2"]] - num = len(choices) - - letter_list = [style.format(letter) for letter in alphabet[0:num]] - if "\t" in style: - letter_list = [letter.replace("\t", "") for letter in letter_list] - - return letter_list - - -# Letters + Full continuation -def choice_C(alphabet, style, doc): - - choices = [doc["sol1"], doc["sol2"]] - num = len(choices) - - letter_list = [style.format(letter) for letter in alphabet[0:num]] - if "\t" not in style: - letter_list = [letter + " " for letter in letter_list] - - return [letter + choice for letter, choice in zip(letter_list, choices)] - - -template_01 = partial(doc_to_text_base, string.ascii_lowercase, "({})") -choice_01a = choice_A -choice_01b = partial(choice_B, string.ascii_lowercase, "({})") -choice_01c = partial(choice_C, string.ascii_lowercase, "({})") -template_02 = partial(doc_to_text_base, string.ascii_lowercase, "{})") -choice_02a = choice_A -choice_02b = partial(choice_B, string.ascii_lowercase, "{})") -choice_02c = partial(choice_C, string.ascii_lowercase, "{})") -template_03 = partial(doc_to_text_base, string.ascii_lowercase, "{}.") -choice_03a = choice_A -choice_03b = partial(choice_B, string.ascii_lowercase, "{}.") -choice_03c = partial(choice_C, string.ascii_lowercase, "{}.") -template_04 = partial(doc_to_text_base, string.ascii_lowercase, "{}\t") -choice_04a = choice_A -choice_04b = partial(choice_B, string.ascii_lowercase, "{}\t") -choice_04c = partial(choice_C, string.ascii_lowercase, "{}\t") -template_05 = partial(doc_to_text_base, string.ascii_uppercase, "({})") -choice_05a = choice_A -choice_05b = partial(choice_B, string.ascii_uppercase, "({})") -choice_05c = partial(choice_C, string.ascii_uppercase, "({})") -template_06 = partial(doc_to_text_base, string.ascii_uppercase, "{})") -choice_06a = choice_A -choice_06b = partial(choice_B, string.ascii_uppercase, "{})") -choice_06c = partial(choice_C, string.ascii_uppercase, "{})") -template_07 = partial(doc_to_text_base, string.ascii_uppercase, "{}.") -choice_07a = choice_A -choice_07b = partial(choice_B, string.ascii_uppercase, "{}.") -choice_07c = partial(choice_C, string.ascii_uppercase, "{}.") -template_08 = partial(doc_to_text_base, string.ascii_uppercase, "{}\t") -choice_08a = choice_A -choice_08b = partial(choice_B, string.ascii_uppercase, "{}\t") -choice_08c = partial(choice_C, string.ascii_uppercase, "{}\t") diff --git a/lm_eval/tasks/piqa/alternative_worlds/prompt_variation/_piqa_yaml b/lm_eval/tasks/piqa/alternative_worlds/prompt_variation/_piqa_yaml deleted file mode 100644 index eb1580c7..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/prompt_variation/_piqa_yaml +++ /dev/null @@ -1,22 +0,0 @@ -dataset_path: piqa -dataset_name: null -output_type: multiple_choice -training_split: train -validation_split: validation -doc_to_text: "Question: {{goal}}\nAnswer:" -doc_to_target: label -doc_to_choice: "{{[sol1, sol2]}}" -should_decontaminate: true -doc_to_decontamination_query: goal -metric_list: - - metric: acc - aggregation: mean - higher_is_better: true - - metric: acc_norm - aggregation: mean - higher_is_better: true - - metric: brier_score - aggregation: brier_score - higher_is_better: false -metadata: - - version: 1.0 diff --git a/lm_eval/tasks/piqa/alternative_worlds/prompt_variation/style_01.yaml b/lm_eval/tasks/piqa/alternative_worlds/prompt_variation/style_01.yaml deleted file mode 100644 index 563c551d..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/prompt_variation/style_01.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: _piqa_yaml -group: piqa_alt_pv -task: piqa_alt_pv_01 -doc_to_text: "{{goal}}" -doc_to_decontamination_query: "{{goal}}" diff --git a/lm_eval/tasks/piqa/alternative_worlds/prompt_variation/style_02.yaml b/lm_eval/tasks/piqa/alternative_worlds/prompt_variation/style_02.yaml deleted file mode 100644 index 32e1fe76..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/prompt_variation/style_02.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: _piqa_yaml -group: piqa_alt_pv -task: piqa_alt_pv_02 -doc_to_text: "Q: {{goal}}\nA:" -doc_to_decontamination_query: "Q: {{goal}}\nA:" diff --git a/lm_eval/tasks/piqa/alternative_worlds/prompt_variation/style_03.yaml b/lm_eval/tasks/piqa/alternative_worlds/prompt_variation/style_03.yaml deleted file mode 100644 index 7bb8bfef..00000000 --- a/lm_eval/tasks/piqa/alternative_worlds/prompt_variation/style_03.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: _piqa_yaml -group: piqa_alt_pv -task: piqa_alt_pv_03 -doc_to_text: "Question: {{goal}}\nAnswer:" -doc_to_decontamination_query: "Question: {{goal}}\nAnswer:" diff --git a/lm_eval/tasks/sciq/alternative_worlds/output_variation/_sciq_alt_yaml b/lm_eval/tasks/sciq/alternative_worlds/output_variation/_sciq_alt_yaml deleted file mode 100644 index f4241098..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/output_variation/_sciq_alt_yaml +++ /dev/null @@ -1,23 +0,0 @@ -group: - - ai2_arc -task: sciq -dataset_path: ai2_arc -dataset_name: ARC-Easy -output_type: multiple_choice -training_split: train -validation_split: validation -test_split: test -doc_to_text: "Question: {{question}}\nAnswer:" -doc_to_target: "{{choices.label.index(answerKey)}}" -doc_to_choice: "{{choices.text}}" -should_decontaminate: true -doc_to_decontamination_query: "Question: {{question}}\nAnswer:" -metric_list: - - metric: acc - aggregation: mean - higher_is_better: true - - metric: acc_norm - aggregation: mean - higher_is_better: true - - metric: brier_score - higher_is_better: false diff --git a/lm_eval/tasks/sciq/alternative_worlds/output_variation/sciq_alt_ov.yaml b/lm_eval/tasks/sciq/alternative_worlds/output_variation/sciq_alt_ov.yaml deleted file mode 100644 index 6810b9db..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/output_variation/sciq_alt_ov.yaml +++ /dev/null @@ -1,10 +0,0 @@ -group: sciq_alt_ov -task: - - sciq_alt_ov_01 - - sciq_alt_ov_02 - - sciq_alt_ov_03 - - sciq_alt_ov_04 - - sciq_alt_ov_05 - - sciq_alt_ov_06 - - sciq_alt_ov_07 - - sciq_alt_ov_08 diff --git a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_01/a.yaml b/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_01/a.yaml deleted file mode 100644 index 4e57348a..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_01/a.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_sciq_alt_yaml -group: sciq_alt_ov_01 -task: sciq_alt_ov_01a -doc_to_text: !function ../styles.template_01 -doc_to_choice: !function ../styles.choice_01a diff --git a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_01/b.yaml b/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_01/b.yaml deleted file mode 100644 index f6f28680..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_01/b.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_sciq_alt_yaml -group: sciq_alt_ov_01 -task: sciq_alt_ov_01b -doc_to_text: !function ../styles.template_01 -doc_to_choice: !function ../styles.choice_01b diff --git a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_01/c.yaml b/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_01/c.yaml deleted file mode 100644 index 49269ec1..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_01/c.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_sciq_alt_yaml -group: sciq_alt_ov_01 -task: sciq_alt_ov_01c -doc_to_text: !function ../styles.template_01 -doc_to_choice: !function ../styles.choice_01c diff --git a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_02/a.yaml b/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_02/a.yaml deleted file mode 100644 index 985a80fa..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_02/a.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_sciq_alt_yaml -group: sciq_alt_ov_02 -task: sciq_alt_ov_02a -doc_to_text: !function ../styles.template_02 -doc_to_choice: !function ../styles.choice_02a diff --git a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_02/b.yaml b/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_02/b.yaml deleted file mode 100644 index 64aa22c7..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_02/b.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_sciq_alt_yaml -group: sciq_alt_ov_02 -task: sciq_alt_ov_02b -doc_to_text: !function ../styles.template_02 -doc_to_choice: !function ../styles.choice_02b diff --git a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_02/c.yaml b/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_02/c.yaml deleted file mode 100644 index 2cf2f59f..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_02/c.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_sciq_alt_yaml -group: sciq_alt_ov_02 -task: sciq_alt_ov_02c -doc_to_text: !function ../styles.template_02 -doc_to_choice: !function ../styles.choice_02c diff --git a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_03/a.yaml b/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_03/a.yaml deleted file mode 100644 index c65ccc19..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_03/a.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_sciq_alt_yaml -group: sciq_alt_ov_03 -task: sciq_alt_ov_03a -doc_to_text: !function ../styles.template_03 -doc_to_choice: !function ../styles.choice_03a diff --git a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_03/b.yaml b/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_03/b.yaml deleted file mode 100644 index 0fc54341..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_03/b.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_sciq_alt_yaml -group: sciq_alt_ov_03 -task: sciq_alt_ov_03b -doc_to_text: !function ../styles.template_03 -doc_to_choice: !function ../styles.choice_03b diff --git a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_03/c.yaml b/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_03/c.yaml deleted file mode 100644 index 1ec41f3b..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_03/c.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_sciq_alt_yaml -group: sciq_alt_ov_03 -task: sciq_alt_ov_03c -doc_to_text: !function ../styles.template_03 -doc_to_choice: !function ../styles.choice_03c diff --git a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_04/a.yaml b/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_04/a.yaml deleted file mode 100644 index 47e929ae..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_04/a.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_sciq_alt_yaml -group: sciq_alt_ov_04 -task: sciq_alt_ov_04a -doc_to_text: !function ../styles.template_04 -doc_to_choice: !function ../styles.choice_04a diff --git a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_04/b.yaml b/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_04/b.yaml deleted file mode 100644 index 805ffa8f..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_04/b.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_sciq_alt_yaml -group: sciq_alt_ov_04 -task: sciq_alt_ov_04b -doc_to_text: !function ../styles.template_04 -doc_to_choice: !function ../styles.choice_04b diff --git a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_04/c.yaml b/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_04/c.yaml deleted file mode 100644 index 11749248..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_04/c.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_sciq_alt_yaml -group: sciq_alt_ov_04 -task: sciq_alt_ov_04c -doc_to_text: !function ../styles.template_04 -doc_to_choice: !function ../styles.choice_04c diff --git a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_05/a.yaml b/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_05/a.yaml deleted file mode 100644 index 1b2e153b..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_05/a.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_sciq_alt_yaml -group: sciq_alt_ov_05 -task: sciq_alt_ov_05a -doc_to_text: !function ../styles.template_05 -doc_to_choice: !function ../styles.choice_05a diff --git a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_05/b.yaml b/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_05/b.yaml deleted file mode 100644 index 0ae6f1fa..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_05/b.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_sciq_alt_yaml -group: sciq_alt_ov_05 -task: sciq_alt_ov_05b -doc_to_text: !function ../styles.template_05 -doc_to_choice: !function ../styles.choice_05b diff --git a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_05/c.yaml b/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_05/c.yaml deleted file mode 100644 index db4dcbfb..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_05/c.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_sciq_alt_yaml -group: sciq_alt_ov_05 -task: sciq_alt_ov_05c -doc_to_text: !function ../styles.template_05 -doc_to_choice: !function ../styles.choice_05c diff --git a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_06/a.yaml b/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_06/a.yaml deleted file mode 100644 index 9f34ac56..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_06/a.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_sciq_alt_yaml -group: sciq_alt_ov_06 -task: sciq_alt_ov_06a -doc_to_text: !function ../styles.template_06 -doc_to_choice: !function ../styles.choice_06a diff --git a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_06/b.yaml b/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_06/b.yaml deleted file mode 100644 index 1b86dbf9..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_06/b.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_sciq_alt_yaml -group: sciq_alt_ov_06 -task: sciq_alt_ov_06b -doc_to_text: !function ../styles.template_06 -doc_to_choice: !function ../styles.choice_06b diff --git a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_06/c.yaml b/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_06/c.yaml deleted file mode 100644 index 5f505f78..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_06/c.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_sciq_alt_yaml -group: sciq_alt_ov_06 -task: sciq_alt_ov_06c -doc_to_text: !function ../styles.template_06 -doc_to_choice: !function ../styles.choice_06c diff --git a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_07/a.yaml b/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_07/a.yaml deleted file mode 100644 index 325ecd90..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_07/a.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_sciq_alt_yaml -group: sciq_alt_ov_07 -task: sciq_alt_ov_07a -doc_to_text: !function ../styles.template_07 -doc_to_choice: !function ../styles.choice_07a diff --git a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_07/b.yaml b/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_07/b.yaml deleted file mode 100644 index cd50fd65..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_07/b.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_sciq_alt_yaml -group: sciq_alt_ov_07 -task: sciq_alt_ov_07b -doc_to_text: !function ../styles.template_07 -doc_to_choice: !function ../styles.choice_07b diff --git a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_07/c.yaml b/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_07/c.yaml deleted file mode 100644 index 46f78886..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_07/c.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_sciq_alt_yaml -group: sciq_alt_ov_07 -task: sciq_alt_ov_07c -doc_to_text: !function ../styles.template_07 -doc_to_choice: !function ../styles.choice_07c diff --git a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_08/a.yaml b/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_08/a.yaml deleted file mode 100644 index d744ce7a..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_08/a.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_sciq_alt_yaml -group: sciq_alt_ov_08 -task: sciq_alt_ov_08a -doc_to_text: !function ../styles.template_08 -doc_to_choice: !function ../styles.choice_08a diff --git a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_08/b.yaml b/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_08/b.yaml deleted file mode 100644 index 885761d5..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_08/b.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_sciq_alt_yaml -group: sciq_alt_ov_08 -task: sciq_alt_ov_08b -doc_to_text: !function ../styles.template_08 -doc_to_choice: !function ../styles.choice_08b diff --git a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_08/c.yaml b/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_08/c.yaml deleted file mode 100644 index 68e6037b..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/output_variation/style_08/c.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: ../_sciq_alt_yaml -group: sciq_alt_ov_08 -task: sciq_alt_ov_08c -doc_to_text: !function ../styles.template_08 -doc_to_choice: !function ../styles.choice_08c diff --git a/lm_eval/tasks/sciq/alternative_worlds/output_variation/styles.py b/lm_eval/tasks/sciq/alternative_worlds/output_variation/styles.py deleted file mode 100644 index 3d075517..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/output_variation/styles.py +++ /dev/null @@ -1,89 +0,0 @@ -import string -from functools import partial - - -def doc_to_text_base(alphabet, style, doc): - - choices = doc["choices"]["text"] - num = len(choices) - - letter_list = [style.format(letter) for letter in alphabet[0:num]] - - if "\t" in style: - choice_string = "{}{}" - else: - choice_string = "{} {}" - - doc_to_text = "\n\n".join( - [ - "Question: " + doc["question"] + "\nAnswer:", - ] - + [choice_string.format(i, j) for i, j in zip(letter_list, choices)] - ) - - return doc_to_text - - -# Full continuation -def choice_A(doc): - return doc["choices"]["text"] - - -# Letters only -def choice_B(alphabet, style, doc): - - choices = doc["choices"]["text"] - num = len(choices) - - letter_list = [style.format(letter) for letter in alphabet[0:num]] - if "\t" in style: - letter_list = [letter.replace("\t", "") for letter in letter_list] - - return letter_list - - -# Letters + Full continuation -def choice_C(alphabet, style, doc): - - choices = doc["choices"]["text"] - num = len(choices) - - letter_list = [style.format(letter) for letter in alphabet[0:num]] - if "\t" not in style: - letter_list = [letter + " " for letter in letter_list] - - return [letter + choice for letter, choice in zip(letter_list, choices)] - - -template_01 = partial(doc_to_text_base, string.ascii_lowercase, "({})") -choice_01a = choice_A -choice_01b = partial(choice_B, string.ascii_lowercase, "({})") -choice_01c = partial(choice_C, string.ascii_lowercase, "({})") -template_02 = partial(doc_to_text_base, string.ascii_lowercase, "{})") -choice_02a = choice_A -choice_02b = partial(choice_B, string.ascii_lowercase, "{})") -choice_02c = partial(choice_C, string.ascii_lowercase, "{})") -template_03 = partial(doc_to_text_base, string.ascii_lowercase, "{}.") -choice_03a = choice_A -choice_03b = partial(choice_B, string.ascii_lowercase, "{}.") -choice_03c = partial(choice_C, string.ascii_lowercase, "{}.") -template_04 = partial(doc_to_text_base, string.ascii_lowercase, "{}\t") -choice_04a = choice_A -choice_04b = partial(choice_B, string.ascii_lowercase, "{}\t") -choice_04c = partial(choice_C, string.ascii_lowercase, "{}\t") -template_05 = partial(doc_to_text_base, string.ascii_uppercase, "({})") -choice_05a = choice_A -choice_05b = partial(choice_B, string.ascii_uppercase, "({})") -choice_05c = partial(choice_C, string.ascii_uppercase, "({})") -template_06 = partial(doc_to_text_base, string.ascii_uppercase, "{})") -choice_06a = choice_A -choice_06b = partial(choice_B, string.ascii_uppercase, "{})") -choice_06c = partial(choice_C, string.ascii_uppercase, "{})") -template_07 = partial(doc_to_text_base, string.ascii_uppercase, "{}.") -choice_07a = choice_A -choice_07b = partial(choice_B, string.ascii_uppercase, "{}.") -choice_07c = partial(choice_C, string.ascii_uppercase, "{}.") -template_08 = partial(doc_to_text_base, string.ascii_uppercase, "{}\t") -choice_08a = choice_A -choice_08b = partial(choice_B, string.ascii_uppercase, "{}\t") -choice_08c = partial(choice_C, string.ascii_uppercase, "{}\t") diff --git a/lm_eval/tasks/sciq/alternative_worlds/prompt_variation/style_01.yaml b/lm_eval/tasks/sciq/alternative_worlds/prompt_variation/style_01.yaml deleted file mode 100644 index c653b2d9..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/prompt_variation/style_01.yaml +++ /dev/null @@ -1,17 +0,0 @@ -group: sciq_alt_pv -task: sciq_alt_pv_01 -dataset_path: sciq -dataset_name: null -output_type: multiple_choice -training_split: train -validation_split: validation -test_split: test -doc_to_text: "{{support.lstrip()}}\n{{question}}" -doc_to_target: 3 -doc_to_choice: "{{[distractor1, distractor2, distractor3, correct_answer]}}" -should_decontaminate: true -doc_to_decontamination_query: "{{support}} {{question}}" -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/sciq/alternative_worlds/prompt_variation/style_02.yaml b/lm_eval/tasks/sciq/alternative_worlds/prompt_variation/style_02.yaml deleted file mode 100644 index a9498269..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/prompt_variation/style_02.yaml +++ /dev/null @@ -1,17 +0,0 @@ -group: sciq_alt_pv -task: sciq_alt_pv_02 -dataset_path: sciq -dataset_name: null -output_type: multiple_choice -training_split: train -validation_split: validation -test_split: test -doc_to_text: "{{support.lstrip()}}\nQ: {{question}}\nA:" -doc_to_target: 3 -doc_to_choice: "{{[distractor1, distractor2, distractor3, correct_answer]}}" -should_decontaminate: true -doc_to_decontamination_query: "{{support}} {{question}}" -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/sciq/alternative_worlds/prompt_variation/style_03.yaml b/lm_eval/tasks/sciq/alternative_worlds/prompt_variation/style_03.yaml deleted file mode 100644 index 12049228..00000000 --- a/lm_eval/tasks/sciq/alternative_worlds/prompt_variation/style_03.yaml +++ /dev/null @@ -1,17 +0,0 @@ -group: sciq_alt_pv -task: sciq_alt_pv_03 -dataset_path: sciq -dataset_name: null -output_type: multiple_choice -training_split: train -validation_split: validation -test_split: test -doc_to_text: "{{support.lstrip()}}\nQuestion: {{question}}\nAnswer:" -doc_to_target: 3 -doc_to_choice: "{{[distractor1, distractor2, distractor3, correct_answer]}}" -should_decontaminate: true -doc_to_decontamination_query: "{{support}} {{question}}" -metric_list: - - metric: acc - - metric: acc_norm - - metric: brier_score diff --git a/lm_eval/tasks/siqa/alternative_worlds/README.md b/lm_eval/tasks/siqa/alternative_worlds/README.md deleted file mode 100644 index 93600ae1..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/README.md +++ /dev/null @@ -1,20 +0,0 @@ - - -Investigate affect of letter options -- (A) -- A) -- A. -- A\t -- (a) -- a) -- a. -- a\t - -Answer types: -- letters only - - original option - - just letter -- letters + continuation - - original option - - just letter -- continuation diff --git a/lm_eval/tasks/siqa/alternative_worlds/output_variation/_siqa_alt_yaml b/lm_eval/tasks/siqa/alternative_worlds/output_variation/_siqa_alt_yaml deleted file mode 100644 index 157c7389..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/output_variation/_siqa_alt_yaml +++ /dev/null @@ -1,19 +0,0 @@ -task: social_iqa -dataset_path: social_i_qa -dataset_name: null -output_type: multiple_choice -training_split: train -validation_split: validation -doc_to_choice: ["{{answerA}}", "{{answerB}}", "{{answerC}}"] -doc_to_target: "{{label}}" -should_decontaminate: true -metric_list: - - metric: acc - aggregation: mean - higher_is_better: true - - metric: acc_norm - aggregation: mean - higher_is_better: true - - metric: brier_score - aggregation: brier_score - higher_is_better: false diff --git a/lm_eval/tasks/siqa/alternative_worlds/output_variation/siqa_alt.yaml b/lm_eval/tasks/siqa/alternative_worlds/output_variation/siqa_alt.yaml deleted file mode 100644 index 4d7079ce..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/output_variation/siqa_alt.yaml +++ /dev/null @@ -1,10 +0,0 @@ -group: siqa_alt_ov -task: - - siqa_alt_ov_01 - - siqa_alt_ov_02 - - siqa_alt_ov_03 - - siqa_alt_ov_04 - - siqa_alt_ov_05 - - siqa_alt_ov_06 - - siqa_alt_ov_07 - - siqa_alt_ov_08 diff --git a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_01/a.yaml b/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_01/a.yaml deleted file mode 100644 index 9d791740..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_01/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_siqa_alt_yaml -group: siqa_alt_ov_01 -task: siqa_alt_ov_01a -doc_to_text: !function ../styles.template_01 -doc_to_choice: !function ../styles.choice_01a -doc_to_decontamination_query: !function ../styles.template_01 diff --git a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_01/b.yaml b/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_01/b.yaml deleted file mode 100644 index b942970f..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_01/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_siqa_alt_yaml -group: siqa_alt_ov_01 -task: siqa_alt_ov_01b -doc_to_text: !function ../styles.template_01 -doc_to_choice: !function ../styles.choice_01b -doc_to_decontamination_query: !function ../styles.template_01 diff --git a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_01/c.yaml b/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_01/c.yaml deleted file mode 100644 index a4bb701d..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_01/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_siqa_alt_yaml -group: siqa_alt_ov_01 -task: siqa_alt_ov_01c -doc_to_text: !function ../styles.template_01 -doc_to_choice: !function ../styles.choice_01c -doc_to_decontamination_query: !function ../styles.template_01 diff --git a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_02/a.yaml b/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_02/a.yaml deleted file mode 100644 index 94477b97..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_02/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_siqa_alt_yaml -group: siqa_alt_ov_02 -task: siqa_alt_ov_02a -doc_to_text: !function ../styles.template_02 -doc_to_choice: !function ../styles.choice_02a -doc_to_decontamination_query: !function ../styles.template_02 diff --git a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_02/b.yaml b/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_02/b.yaml deleted file mode 100644 index a57d7d15..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_02/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_siqa_alt_yaml -group: siqa_alt_ov_02 -task: siqa_alt_ov_02b -doc_to_text: !function ../styles.template_02 -doc_to_choice: !function ../styles.choice_02b -doc_to_decontamination_query: !function ../styles.template_02 diff --git a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_02/c.yaml b/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_02/c.yaml deleted file mode 100644 index 5a47424a..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_02/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_siqa_alt_yaml -group: siqa_alt_ov_02 -task: siqa_alt_ov_02c -doc_to_text: !function ../styles.template_02 -doc_to_choice: !function ../styles.choice_02c -doc_to_decontamination_query: !function ../styles.template_02 diff --git a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_03/a.yaml b/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_03/a.yaml deleted file mode 100644 index 5eeabdd3..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_03/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_siqa_alt_yaml -group: siqa_alt_ov_03 -task: siqa_alt_ov_03a -doc_to_text: !function ../styles.template_03 -doc_to_choice: !function ../styles.choice_03a -doc_to_decontamination_query: !function ../styles.template_03 diff --git a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_03/b.yaml b/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_03/b.yaml deleted file mode 100644 index 0b39ea71..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_03/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_siqa_alt_yaml -group: siqa_alt_ov_03 -task: siqa_alt_ov_03b -doc_to_text: !function ../styles.template_03 -doc_to_choice: !function ../styles.choice_03b -doc_to_decontamination_query: !function ../styles.template_03 diff --git a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_03/c.yaml b/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_03/c.yaml deleted file mode 100644 index cf97679a..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_03/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_siqa_alt_yaml -group: siqa_alt_ov_03 -task: siqa_alt_ov_03c -doc_to_text: !function ../styles.template_03 -doc_to_choice: !function ../styles.choice_03c -doc_to_decontamination_query: !function ../styles.template_03 diff --git a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_04/a.yaml b/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_04/a.yaml deleted file mode 100644 index 251cf948..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_04/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_siqa_alt_yaml -group: siqa_alt_ov_04 -task: siqa_alt_ov_04a -doc_to_text: !function ../styles.template_04 -doc_to_choice: !function ../styles.choice_04a -doc_to_decontamination_query: !function ../styles.template_04 diff --git a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_04/b.yaml b/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_04/b.yaml deleted file mode 100644 index ea9619a7..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_04/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_siqa_alt_yaml -group: siqa_alt_ov_04 -task: siqa_alt_ov_04b -doc_to_text: !function ../styles.template_04 -doc_to_choice: !function ../styles.choice_04b -doc_to_decontamination_query: !function ../styles.template_04 diff --git a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_04/c.yaml b/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_04/c.yaml deleted file mode 100644 index 8cacf98b..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_04/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_siqa_alt_yaml -group: siqa_alt_ov_04 -task: siqa_alt_ov_04c -doc_to_text: !function ../styles.template_04 -doc_to_choice: !function ../styles.choice_04c -doc_to_decontamination_query: !function ../styles.template_04 diff --git a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_05/a.yaml b/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_05/a.yaml deleted file mode 100644 index 4753ad0b..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_05/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_siqa_alt_yaml -group: siqa_alt_ov_05 -task: siqa_alt_ov_05a -doc_to_text: !function ../styles.template_05 -doc_to_choice: !function ../styles.choice_05a -doc_to_decontamination_query: !function ../styles.template_05 diff --git a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_05/b.yaml b/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_05/b.yaml deleted file mode 100644 index 0ae84f2d..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_05/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_siqa_alt_yaml -group: siqa_alt_ov_05 -task: siqa_alt_ov_05b -doc_to_text: !function ../styles.template_05 -doc_to_choice: !function ../styles.choice_05b -doc_to_decontamination_query: !function ../styles.template_05 diff --git a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_05/c.yaml b/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_05/c.yaml deleted file mode 100644 index fd47fb6b..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_05/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_siqa_alt_yaml -group: siqa_alt_ov_05 -task: siqa_alt_ov_05c -doc_to_text: !function ../styles.template_05 -doc_to_choice: !function ../styles.choice_05c -doc_to_decontamination_query: !function ../styles.template_05 diff --git a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_06/a.yaml b/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_06/a.yaml deleted file mode 100644 index 12774974..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_06/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_siqa_alt_yaml -group: siqa_alt_ov_06 -task: siqa_alt_ov_06a -doc_to_text: !function ../styles.template_06 -doc_to_choice: !function ../styles.choice_06a -doc_to_decontamination_query: !function ../styles.template_06 diff --git a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_06/b.yaml b/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_06/b.yaml deleted file mode 100644 index 30cbf0ef..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_06/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_siqa_alt_yaml -group: siqa_alt_ov_06 -task: siqa_alt_ov_06b -doc_to_text: !function ../styles.template_06 -doc_to_choice: !function ../styles.choice_06b -doc_to_decontamination_query: !function ../styles.template_06 diff --git a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_06/c.yaml b/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_06/c.yaml deleted file mode 100644 index f309834d..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_06/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_siqa_alt_yaml -group: siqa_alt_ov_06 -task: siqa_alt_ov_06c -doc_to_text: !function ../styles.template_06 -doc_to_choice: !function ../styles.choice_06c -doc_to_decontamination_query: !function ../styles.template_06 diff --git a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_07/a.yaml b/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_07/a.yaml deleted file mode 100644 index 5cc468f9..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_07/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_siqa_alt_yaml -group: siqa_alt_ov_07 -task: siqa_alt_ov_07a -doc_to_text: !function ../styles.template_07 -doc_to_choice: !function ../styles.choice_07a -doc_to_decontamination_query: !function ../styles.template_07 diff --git a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_07/b.yaml b/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_07/b.yaml deleted file mode 100644 index e7de1319..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_07/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_siqa_alt_yaml -group: siqa_alt_ov_07 -task: siqa_alt_ov_07b -doc_to_text: !function ../styles.template_07 -doc_to_choice: !function ../styles.choice_07b -doc_to_decontamination_query: !function ../styles.template_07 diff --git a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_07/c.yaml b/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_07/c.yaml deleted file mode 100644 index b0904d14..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_07/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_siqa_alt_yaml -group: siqa_alt_ov_07 -task: siqa_alt_ov_07c -doc_to_text: !function ../styles.template_07 -doc_to_choice: !function ../styles.choice_07c -doc_to_decontamination_query: !function ../styles.template_07 diff --git a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_08/a.yaml b/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_08/a.yaml deleted file mode 100644 index 0d971d4e..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_08/a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_siqa_alt_yaml -group: siqa_alt_ov_08 -task: siqa_alt_ov_08a -doc_to_text: !function ../styles.template_08 -doc_to_choice: !function ../styles.choice_08a -doc_to_decontamination_query: !function ../styles.template_08 diff --git a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_08/b.yaml b/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_08/b.yaml deleted file mode 100644 index 32151413..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_08/b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_siqa_alt_yaml -group: siqa_alt_ov_08 -task: siqa_alt_ov_08b -doc_to_text: !function ../styles.template_08 -doc_to_choice: !function ../styles.choice_08b -doc_to_decontamination_query: !function ../styles.template_08 diff --git a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_08/c.yaml b/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_08/c.yaml deleted file mode 100644 index 10b32b01..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/output_variation/style_08/c.yaml +++ /dev/null @@ -1,6 +0,0 @@ -include: ../_siqa_alt_yaml -group: siqa_alt_ov_08 -task: siqa_alt_ov_08c -doc_to_text: !function ../styles.template_08 -doc_to_choice: !function ../styles.choice_08c -doc_to_decontamination_query: !function ../styles.template_08 diff --git a/lm_eval/tasks/siqa/alternative_worlds/output_variation/styles.py b/lm_eval/tasks/siqa/alternative_worlds/output_variation/styles.py deleted file mode 100644 index ee263cc5..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/output_variation/styles.py +++ /dev/null @@ -1,90 +0,0 @@ -import string -from functools import partial - - -def doc_to_text_base(alphabet, style, doc): - - choices = [doc["answerA"], doc["answerB"], doc["answerC"]] - num = len(choices) - - letter_list = [style.format(letter) for letter in alphabet[0:num]] - - if "\t" in style: - choice_string = "{}{}" - else: - choice_string = "{} {}" - - doc_to_text = "\n".join( - [ - "Question: " + doc["context"] + " " + doc["question"], - ] - + [choice_string.format(i, j) for i, j in zip(letter_list, choices)] - + ["Answer:"] - ) - - return doc_to_text - - -# Full continuation -def choice_A(doc): - return doc["choices"]["text"] - - -# Letters only -def choice_B(alphabet, style, doc): - - choices = [doc["answerA"], doc["answerB"], doc["answerC"]] - num = len(choices) - - letter_list = [style.format(letter) for letter in alphabet[0:num]] - if "\t" in style: - letter_list = [letter.replace("\t", "") for letter in letter_list] - - return letter_list - - -# Letters + Full continuation -def choice_C(alphabet, style, doc): - - choices = [doc["answerA"], doc["answerB"], doc["answerC"]] - num = len(choices) - - letter_list = [style.format(letter) for letter in alphabet[0:num]] - if "\t" not in style: - letter_list = [letter + " " for letter in letter_list] - - return [letter + choice for letter, choice in zip(letter_list, choices)] - - -template_01 = partial(doc_to_text_base, string.ascii_lowercase, "({})") -choice_01a = choice_A -choice_01b = partial(choice_B, string.ascii_lowercase, "({})") -choice_01c = partial(choice_C, string.ascii_lowercase, "({})") -template_02 = partial(doc_to_text_base, string.ascii_lowercase, "{})") -choice_02a = choice_A -choice_02b = partial(choice_B, string.ascii_lowercase, "{})") -choice_02c = partial(choice_C, string.ascii_lowercase, "{})") -template_03 = partial(doc_to_text_base, string.ascii_lowercase, "{}.") -choice_03a = choice_A -choice_03b = partial(choice_B, string.ascii_lowercase, "{}.") -choice_03c = partial(choice_C, string.ascii_lowercase, "{}.") -template_04 = partial(doc_to_text_base, string.ascii_lowercase, "{}\t") -choice_04a = choice_A -choice_04b = partial(choice_B, string.ascii_lowercase, "{}\t") -choice_04c = partial(choice_C, string.ascii_lowercase, "{}\t") -template_05 = partial(doc_to_text_base, string.ascii_uppercase, "({})") -choice_05a = choice_A -choice_05b = partial(choice_B, string.ascii_uppercase, "({})") -choice_05c = partial(choice_C, string.ascii_uppercase, "({})") -template_06 = partial(doc_to_text_base, string.ascii_uppercase, "{})") -choice_06a = choice_A -choice_06b = partial(choice_B, string.ascii_uppercase, "{})") -choice_06c = partial(choice_C, string.ascii_uppercase, "{})") -template_07 = partial(doc_to_text_base, string.ascii_uppercase, "{}.") -choice_07a = choice_A -choice_07b = partial(choice_B, string.ascii_uppercase, "{}.") -choice_07c = partial(choice_C, string.ascii_uppercase, "{}.") -template_08 = partial(doc_to_text_base, string.ascii_uppercase, "{}\t") -choice_08a = choice_A -choice_08b = partial(choice_B, string.ascii_uppercase, "{}\t") -choice_08c = partial(choice_C, string.ascii_uppercase, "{}\t") diff --git a/lm_eval/tasks/siqa/alternative_worlds/prompt_variation/_siqa_alt_yaml b/lm_eval/tasks/siqa/alternative_worlds/prompt_variation/_siqa_alt_yaml deleted file mode 100644 index 157c7389..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/prompt_variation/_siqa_alt_yaml +++ /dev/null @@ -1,19 +0,0 @@ -task: social_iqa -dataset_path: social_i_qa -dataset_name: null -output_type: multiple_choice -training_split: train -validation_split: validation -doc_to_choice: ["{{answerA}}", "{{answerB}}", "{{answerC}}"] -doc_to_target: "{{label}}" -should_decontaminate: true -metric_list: - - metric: acc - aggregation: mean - higher_is_better: true - - metric: acc_norm - aggregation: mean - higher_is_better: true - - metric: brier_score - aggregation: brier_score - higher_is_better: false diff --git a/lm_eval/tasks/siqa/alternative_worlds/prompt_variation/style_01.yaml b/lm_eval/tasks/siqa/alternative_worlds/prompt_variation/style_01.yaml deleted file mode 100644 index 81cc8595..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/prompt_variation/style_01.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: _siqa_alt_yaml -group: siqa_alt_pv -task: siqa_alt_pv_01 -doc_to_text: "{{context}} {{question}}" -doc_to_decontamination_query: "{{context}} {{question}}" diff --git a/lm_eval/tasks/siqa/alternative_worlds/prompt_variation/style_02.yaml b/lm_eval/tasks/siqa/alternative_worlds/prompt_variation/style_02.yaml deleted file mode 100644 index 287bd5fd..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/prompt_variation/style_02.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: _siqa_alt_yaml -group: siqa_alt_pv -task: siqa_alt_pv_02 -doc_to_text: "Q: {{context}} {{question}}\nA:" -doc_to_decontamination_query: "Q: {{context}} {{question}}\nA:" diff --git a/lm_eval/tasks/siqa/alternative_worlds/prompt_variation/style_03.yaml b/lm_eval/tasks/siqa/alternative_worlds/prompt_variation/style_03.yaml deleted file mode 100644 index 76d62c4c..00000000 --- a/lm_eval/tasks/siqa/alternative_worlds/prompt_variation/style_03.yaml +++ /dev/null @@ -1,5 +0,0 @@ -include: _siqa_alt_yaml -group: siqa_alt_pv -task: siqa_alt_pv_03 -doc_to_text: "Question: {{context}} {{question}}\nAnswer:" -doc_to_decontamination_query: "Question: {{context}} {{question}}\nAnswer:" -- GitLab