Merge branch 'group-agg-rework' of...

Merge branch 'group-agg-rework' of https://github.com/EleutherAI/lm-evaluation-harness into group-agg-rework

lm_eval/tasks/arabicmmlu/arabicmmlu_primary_natural_science.yaml

 "dataset_name": "Primary Natural Science"
-"group": "arabicmmlu_stem"
-"group_alias": "stem"
+"tag": "arabicmmlu_stem_tasks"
 "include": "_default_template_yaml"
 "task": "arabicmmlu_primary_natural_science"
 "task_alias": "Primary Natural Science"

lm_eval/tasks/arabicmmlu/arabicmmlu_primary_social_science.yaml

 "dataset_name": "Primary Social Science"
-"group": "arabicmmlu_social_science"
-"group_alias": "social science"
+"tag": "arabicmmlu_social_science_tasks"
 "include": "_default_template_yaml"
 "task": "arabicmmlu_primary_social_science"
 "task_alias": "Primary Social Science"

lm_eval/tasks/arabicmmlu/arabicmmlu_prof_law.yaml

 "dataset_name": "Prof Law"
-"group": "arabicmmlu_humanities"
-"group_alias": "humanities"
+"tag": "arabicmmlu_humanities_tasks"
 "include": "_default_template_yaml"
 "task": "arabicmmlu_prof_law"
 "task_alias": "Prof Law"

lm_eval/tasks/arabicmmlu/arabicmmlu_univ_accounting.yaml

 "dataset_name": "Univ Accounting"
-"group": "arabicmmlu_social_science"
-"group_alias": "social science"
+"tag": "arabicmmlu_social_science_tasks"
 "include": "_default_template_yaml"
 "task": "arabicmmlu_univ_accounting"
 "task_alias": "Univ Accounting"

lm_eval/tasks/arabicmmlu/arabicmmlu_univ_computer_science.yaml

 "dataset_name": "Univ Computer Science"
-"group": "arabicmmlu_stem"
-"group_alias": "stem"
+"tag": "arabicmmlu_stem_tasks"
 "include": "_default_template_yaml"
 "task": "arabicmmlu_univ_computer_science"
 "task_alias": "Univ Computer Science"

lm_eval/tasks/arabicmmlu/arabicmmlu_univ_economics.yaml

 "dataset_name": "Univ Economics"
-"group": "arabicmmlu_social_science"
-"group_alias": "social science"
+"tag": "arabicmmlu_social_science_tasks"
 "include": "_default_template_yaml"
 "task": "arabicmmlu_univ_economics"
 "task_alias": "Univ Economics"

lm_eval/tasks/arabicmmlu/arabicmmlu_univ_management.yaml

 "dataset_name": "Univ Management"
-"group": "arabicmmlu_other"
-"group_alias": "other"
+"tag": "arabicmmlu_other_tasks"
 "include": "_default_template_yaml"
 "task": "arabicmmlu_univ_management"
 "task_alias": "Univ Management"

lm_eval/tasks/arabicmmlu/arabicmmlu_univ_political_science.yaml

 "dataset_name": "Univ Political Science"
-"group": "arabicmmlu_social_science"
-"group_alias": "social science"
+"tag": "arabicmmlu_social_science_tasks"
 "include": "_default_template_yaml"
 "task": "arabicmmlu_univ_political_science"
 "task_alias": "Univ Political Science"

lm_eval/tasks/arc_mt/arc_challenge_mt_fi.yaml

-group:
+tag:
   - arc_challenge_mt
 task: arc_challenge_mt_fi
 dataset_path: LumiOpen/arc_challenge_mt

lm_eval/tasks/basqueglue/bec.yaml

-group: basque-glue
+tag: basque-glue
 task: bec2016eu
 dataset_path: orai-nlp/basqueGLUE
 dataset_name: bec

lm_eval/tasks/basqueglue/bhtc.yaml

-group: basque-glue
+tag: basque-glue
 task: bhtc_v2
 dataset_path: orai-nlp/basqueGLUE
 dataset_name: bhtc

lm_eval/tasks/basqueglue/coref.yaml

-group: basque-glue
+tag: basque-glue
 task: epec_koref_bin
 dataset_path: orai-nlp/basqueGLUE
 dataset_name: coref

lm_eval/tasks/basqueglue/qnli.yaml

-group: basque-glue
+tag: basque-glue
 task: qnlieu
 dataset_path: orai-nlp/basqueGLUE
 dataset_name: qnli

lm_eval/tasks/basqueglue/vaxx.yaml

-group: basque-glue
+tag: basque-glue
 task: vaxx_stance
 dataset_path: orai-nlp/basqueGLUE
 dataset_name: vaxx

lm_eval/tasks/basqueglue/wic.yaml

-group: basque-glue
+tag: basque-glue
 task: wiceu
 dataset_path: orai-nlp/basqueGLUE
 dataset_name: wic

lm_eval/tasks/bbh/cot_fewshot/_bbh.yaml

@@ -5,7 +5,7 @@ task:
   - bbh_cot_fewshot_date_understanding
   - bbh_cot_fewshot_disambiguation_qa
   - bbh_cot_fewshot_dyck_languages
-  - bbh_cot_fewshot_formal_languages
+  - bbh_cot_fewshot_formal_fallacies
   - bbh_cot_fewshot_geometric_shapes
   - bbh_cot_fewshot_hyperbaton
   - bbh_cot_fewshot_logical_deduction_five_objects

lm_eval/tasks/bertaqa/_bertaqa_template

-group: bertaqa
+tag: bertaqa
 dataset_path: HiTZ/BertaQA
 dataset_name: null
 validation_split: null

lm_eval/tasks/inverse_scaling/_inverse_scaling_mc_yaml

-group:
+tag:
   - inverse_scaling_mc
 output_type: multiple_choice
 test_split: train

lm_eval/tasks/inverse_scaling/_some_results

+# |                   Tasks                   |Version|Filter|n-shot| Metric |Value |   |Stderr|
+# |-------------------------------------------|-------|------|-----:|--------|-----:|---|-----:|
+# | - inverse_scaling_hindsight_neglect_10shot|      0|none  |     0|acc     |0.4476|±  |0.0281|
+# |                                           |       |none  |     0|acc_norm|0.4476|±  |0.0281|
+# |inverse_scaling_mc                         |N/A    |none  |     0|acc_norm|0.6273|±  |0.0096|
+# |                                           |       |none  |     0|acc     |0.6210|±  |0.0095|
+# | - inverse_scaling_neqa                    |      0|none  |     0|acc     |0.5300|±  |0.0289|
+# |                                           |       |none  |     0|acc_norm|0.5300|±  |0.0289|
+# | - inverse_scaling_quote_repetition        |      0|none  |     0|acc     |0.9367|±  |0.0141|
+# |                                           |       |none  |     0|acc_norm|0.9367|±  |0.0141|
+# | - inverse_scaling_redefine_math           |      0|none  |     0|acc     |0.7178|±  |0.0150|
+# |                                           |       |none  |     0|acc_norm|0.7178|±  |0.0150|
+# | - inverse_scaling_winobias_antistereotype |      0|none  |     0|acc     |0.3786|±  |0.0239|
+# |                                           |       |none  |     0|acc_norm|0.4126|±  |0.0243|
+
+# |      Groups      |Version|Filter|n-shot| Metric |Value |   |Stderr|
+# |------------------|-------|------|-----:|--------|-----:|---|-----:|
+# |inverse_scaling_mc|N/A    |none  |     0|acc_norm|0.6273|±  |0.0096|
+# |                  |       |none  |     0|acc     |0.6210|±  |0.0095|
+# hf (pretrained=facebook/opt-2.7b,add_bos_token=True,dtype=float32), gen_kwargs: (None), limit: None, num_fewshot: None, batch_size: auto (32)
+# |                   Tasks                   |Version|Filter|n-shot| Metric |Value |   |Stderr|
+# |-------------------------------------------|-------|------|-----:|--------|-----:|---|-----:|
+# | - inverse_scaling_hindsight_neglect_10shot|      0|none  |     0|acc     |0.4476|±  |0.0281|
+# |                                           |       |none  |     0|acc_norm|0.4476|±  |0.0281|
+# |inverse_scaling_mc                         |N/A    |none  |     0|acc_norm|0.6291|±  |0.0095|
+# |                                           |       |none  |     0|acc     |0.6219|±  |0.0095|
+# | - inverse_scaling_neqa                    |      0|none  |     0|acc     |0.5267|±  |0.0289|
+# |                                           |       |none  |     0|acc_norm|0.5267|±  |0.0289|
+# | - inverse_scaling_quote_repetition        |      0|none  |     0|acc     |0.9433|±  |0.0134|
+# |                                           |       |none  |     0|acc_norm|0.9433|±  |0.0134|
+# | - inverse_scaling_redefine_math           |      0|none  |     0|acc     |0.7200|±  |0.0150|
+# |                                           |       |none  |     0|acc_norm|0.7200|±  |0.0150|
+# | - inverse_scaling_winobias_antistereotype |      0|none  |     0|acc     |0.3762|±  |0.0239|
+# |                                           |       |none  |     0|acc_norm|0.4150|±  |0.0243|
+
+# |      Groups      |Version|Filter|n-shot| Metric |Value |   |Stderr|
+# |------------------|-------|------|-----:|--------|-----:|---|-----:|
+# |inverse_scaling_mc|N/A    |none  |     0|acc_norm|0.6291|±  |0.0095|
+# |                  |       |none  |     0|acc     |0.6219|±  |0.0095|

lm_eval/tasks/leaderboard/bbh_mc/_fewshot_template_yaml

-group: leaderboard_bbh
 dataset_path: SaylorTwift/bbh
 output_type: multiple_choice
 test_split: test