Merge branch 'main' into multimodal-prototyping

lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_electrical_engineering.yaml

@@ -42,6 +42,6 @@ fewshot_config:
     target: 'Let''s think step by step. In lap winding, effectively two resistors
       are connected in parallel, so the actual resistance of each pair is 1 Ohm. Since
       we have 50 pairs, we get a total resistance of 50 Ohms. The answer is (C).'
-group: mmlu_flan_cot_fewshot_stem
+tag: mmlu_flan_cot_fewshot_stem
 include: _mmlu_flan_cot_fewshot_template_yaml
 task: mmlu_flan_cot_fewshot_electrical_engineering

lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_elementary_mathematics.yaml

@@ -72,6 +72,6 @@ fewshot_config:
       (D) (5 x 9) x (6 x 9)'
     target: 'Let''s think step by step. We know that 9 = (5 + 4), so 5 x 9 = 5 x (5
       + 4) = (5 x 5) + (5 x 4). The answer is (B).'
-group: mmlu_flan_cot_fewshot_stem
+tag: mmlu_flan_cot_fewshot_stem
 include: _mmlu_flan_cot_fewshot_template_yaml
 task: mmlu_flan_cot_fewshot_elementary_mathematics

lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_formal_logic.yaml

@@ -65,6 +65,6 @@ fewshot_config:
       \ p do not drive on Mars.\nOf all these options, Option (C) appears to be the\
       \ best and most meaningful interpretation of the argument \u201CNo people drive\
       \ on Mars.\u201D The answer is (C).\n\n"
-group: mmlu_flan_cot_fewshot_humanities
+tag: mmlu_flan_cot_fewshot_humanities
 include: _mmlu_flan_cot_fewshot_template_yaml
 task: mmlu_flan_cot_fewshot_formal_logic

lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_global_facts.yaml

@@ -44,6 +44,6 @@ fewshot_config:
       for help. As of 2019, most people tend to be optimistic about their own future
       but pessimistic about the future of their nation or the world. The answer is
       (B).'
-group: mmlu_flan_cot_fewshot_other
+tag: mmlu_flan_cot_fewshot_other
 include: _mmlu_flan_cot_fewshot_template_yaml
 task: mmlu_flan_cot_fewshot_global_facts

lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_biology.yaml

@@ -64,6 +64,6 @@ fewshot_config:
       core cell cycle regulators inside the cell. The most common regulators are cyclins
       and cyclin-dependent kinases. Fibroblast cells do not play any role in cell
       division. The answer is (D).'
-group: mmlu_flan_cot_fewshot_stem
+tag: mmlu_flan_cot_fewshot_stem
 include: _mmlu_flan_cot_fewshot_template_yaml
 task: mmlu_flan_cot_fewshot_high_school_biology

lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_chemistry.yaml

@@ -61,6 +61,6 @@ fewshot_config:
       \ strong acid, Nitric acid, will react with the conjugate base. Therefore the\
       \ maximum amount of acid that can be added will be equal to the amount of acetate\
       \ ion, or 2 moles. The answer is (C).\n\n"
-group: mmlu_flan_cot_fewshot_stem
+tag: mmlu_flan_cot_fewshot_stem
 include: _mmlu_flan_cot_fewshot_template_yaml
 task: mmlu_flan_cot_fewshot_high_school_chemistry

lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_computer_science.yaml

@@ -79,6 +79,6 @@ fewshot_config:
       its value is greater than 100, regardless of the elements in the list. Choice
       D is incorrect because its step 3 does not increment the value of position,
       so it will repeat forever. The answer is (B).'
-group: mmlu_flan_cot_fewshot_stem
+tag: mmlu_flan_cot_fewshot_stem
 include: _mmlu_flan_cot_fewshot_template_yaml
 task: mmlu_flan_cot_fewshot_high_school_computer_science

lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_european_history.yaml

@@ -194,6 +194,6 @@ fewshot_config:
       wrote extensively against the monoplization of power and advocated for a system
       of checks and balances in government to prevent the rise of despotism. The answer
       is (B).'
-group: mmlu_flan_cot_fewshot_humanities
+tag: mmlu_flan_cot_fewshot_humanities
 include: _mmlu_flan_cot_fewshot_template_yaml
 task: mmlu_flan_cot_fewshot_high_school_european_history

lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_geography.yaml

@@ -48,6 +48,6 @@ fewshot_config:
     target: 'Let''s think step by step. We refer to Wikipedia articles on geography
       for help. The difference between number of births and deaths gives the population
       increase at any given time. The answer is (A).'
-group: mmlu_flan_cot_fewshot_social_sciences
+tag: mmlu_flan_cot_fewshot_social_sciences
 include: _mmlu_flan_cot_fewshot_template_yaml
 task: mmlu_flan_cot_fewshot_high_school_geography

lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_government_and_politics.yaml

@@ -56,6 +56,6 @@ fewshot_config:
     target: 'Let''s think step by step. We refer to Wikipedia articles on government
       and politics for help. The US Constitution is not very specific about the powers
       of the president, leading to uncertainty over its limits. The answer is (A).'
-group: mmlu_flan_cot_fewshot_social_sciences
+tag: mmlu_flan_cot_fewshot_social_sciences
 include: _mmlu_flan_cot_fewshot_template_yaml
 task: mmlu_flan_cot_fewshot_high_school_government_and_politics

lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_macroeconomics.yaml

@@ -48,6 +48,6 @@ fewshot_config:
       for help. The economic transactions related to the performance of the American
       pop-singer in Paris happens entirely outside the U.S. and hence is not included
       in the GDP numbers. The answer is (C).'
-group: mmlu_flan_cot_fewshot_social_sciences
+tag: mmlu_flan_cot_fewshot_social_sciences
 include: _mmlu_flan_cot_fewshot_template_yaml
 task: mmlu_flan_cot_fewshot_high_school_macroeconomics

lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_mathematics.yaml

@@ -46,6 +46,6 @@ fewshot_config:
     target: 'Let''s think step by step. The least common multiple of 2, 3 and 5 is
       30, so during a 7 minute dance, all the three lights will come on at the same
       time $2*7+1=15$ times. The answer is (B).'
-group: mmlu_flan_cot_fewshot_stem
+tag: mmlu_flan_cot_fewshot_stem
 include: _mmlu_flan_cot_fewshot_template_yaml
 task: mmlu_flan_cot_fewshot_high_school_mathematics

lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_microeconomics.yaml

@@ -51,6 +51,6 @@ fewshot_config:
       for help. An increase in the construction of new houses means an increase demand
       of in-house painting, thus increases the demand for housepainters. The answer
       is (C).'
-group: mmlu_flan_cot_fewshot_social_sciences
+tag: mmlu_flan_cot_fewshot_social_sciences
 include: _mmlu_flan_cot_fewshot_template_yaml
 task: mmlu_flan_cot_fewshot_high_school_microeconomics

lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_physics.yaml

@@ -45,6 +45,6 @@ fewshot_config:
       cannot have any net displacement because the pipe closure stops them. So the
       particle displacement is at a node. This closure also causes the pressure to
       be maximal, i.e. an antinode. The answer is (B).'
-group: mmlu_flan_cot_fewshot_stem
+tag: mmlu_flan_cot_fewshot_stem
 include: _mmlu_flan_cot_fewshot_template_yaml
 task: mmlu_flan_cot_fewshot_high_school_physics

lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_psychology.yaml

@@ -59,6 +59,6 @@ fewshot_config:
       for help. People with an external locus of control believes fate and luck play
       an important role in their lives, while people with an internal locus of control
       believes they control their lives. The answer is (D).'
-group: mmlu_flan_cot_fewshot_social_sciences
+tag: mmlu_flan_cot_fewshot_social_sciences
 include: _mmlu_flan_cot_fewshot_template_yaml
 task: mmlu_flan_cot_fewshot_high_school_psychology

lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_statistics.yaml

@@ -76,6 +76,6 @@ fewshot_config:
       not perfectly correlated. Statement B is false because uncorrelated variables
       regression lines can have slope zero. Statement C is false because correlation
       is symmetric in the two random variables. The answer is (D).'
-group: mmlu_flan_cot_fewshot_stem
+tag: mmlu_flan_cot_fewshot_stem
 include: _mmlu_flan_cot_fewshot_template_yaml
 task: mmlu_flan_cot_fewshot_high_school_statistics

lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_us_history.yaml

@@ -151,6 +151,6 @@ fewshot_config:
       suspect Washington''s military response to Whiskey Rebellion. Bacon''s Rebellion
       and Pontiac''s Rebellion happen before the Revolution and they can be ruled
       out. The answer is (C).'
-group: mmlu_flan_cot_fewshot_humanities
+tag: mmlu_flan_cot_fewshot_humanities
 include: _mmlu_flan_cot_fewshot_template_yaml
 task: mmlu_flan_cot_fewshot_high_school_us_history

lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_world_history.yaml

@@ -95,6 +95,6 @@ fewshot_config:
       for help. Brahman refers to the ultimate reality of all things in the Hindu
       religion. In contrast, Buddhism does not have a concept of supreme God. The
       answer is (A).'
-group: mmlu_flan_cot_fewshot_humanities
+tag: mmlu_flan_cot_fewshot_humanities
 include: _mmlu_flan_cot_fewshot_template_yaml
 task: mmlu_flan_cot_fewshot_high_school_world_history

lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_aging.yaml

@@ -37,6 +37,6 @@ fewshot_config:
     target: 'Let''s think step by step. We refer to Wikipedia articles on human aging
       for help. Texas does not have state tax, and has low cost of living compared
       with the other three options. The answer is (A).'
-group: mmlu_flan_cot_fewshot_other
+tag: mmlu_flan_cot_fewshot_other
 include: _mmlu_flan_cot_fewshot_template_yaml
 task: mmlu_flan_cot_fewshot_human_aging

lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_sexuality.yaml

@@ -45,6 +45,6 @@ fewshot_config:
     target: 'Let''s think step by step. We refer to Wikipedia articles on human sexuality
       for help. Morning sickness usually begins by nine weeks after conception, corresponding
       to the first trimester. The answer is (A).'
-group: mmlu_flan_cot_fewshot_social_sciences
+tag: mmlu_flan_cot_fewshot_social_sciences
 include: _mmlu_flan_cot_fewshot_template_yaml
 task: mmlu_flan_cot_fewshot_human_sexuality