removed unused files for now

lm_eval/tasks/mmlu/generative/mmlu_virology.yaml

+"dataset_name": "virology"
+"description": "The following are multiple choice questions (with answers) about virology.\n\
+  \n"
+"group": "mmlu_other_generative"
+"group_alias": "other"
+"include": "_default_template_yaml"
+"task": "mmlu_virology_generative"
+"task_alias": "virology"

lm_eval/tasks/mmlu/generative/mmlu_world_religions.yaml

+"dataset_name": "world_religions"
+"description": "The following are multiple choice questions (with answers) about world\
+  \ religions.\n\n"
+"group": "mmlu_humanities_generative"
+"group_alias": "humanities"
+"include": "_default_template_yaml"
+"task": "mmlu_world_religions_generative"
+"task_alias": "world_religions"

lm_eval/tasks/nq_open/nq_open.yaml

@@ -27,6 +27,6 @@ metric_list:
     ignore_case: true
     ignore_punctuation: true
     regexes_to_ignore:
-    - "\ban|a|the\b"
+    - "\\b(?:The |the |An |A |The |a |an )"
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/polemo2/polemo2_in.yaml

@@ -2,7 +2,7 @@ group:
   - polemo2
 task: polemo2_in
 dataset_path: allegro/klej-polemo2-in
-dataset_name: klej-polemo2-in
+dataset_name: null
 output_type: generate_until
 training_split: train
 validation_split: validation

lm_eval/tasks/super_glue/boolq/alternative_worlds/README.md

-
-
-Investigate affect of letter options
-- (A)
-- A)
-- A.
-- A\t
-- (a)
-- a)
-- a.
-- a\t
-
-Answer types:
-- letters only
-    - original option
-    - just letter
-- letters + continuation
-    - original option
-    - just letter
-- continuation

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/_boolq_alt_ov_yaml

-dataset_path: super_glue
-dataset_name: boolq
-output_type: multiple_choice
-training_split: train
-validation_split: validation
-doc_to_text: "{{passage}}\nQuestion: {{question}}?\nAnswer:"
-doc_to_target: label
-doc_to_choice: ["no", "yes"]
-should_decontaminate: true
-doc_to_decontamination_query: passage
-  - metric: acc
-    aggregation: mean
-    higher_is_better: true
-  - metric: acc_norm
-    aggregation: mean
-    higher_is_better: true
-  - metric: brier_score
-    aggregation: brier_score
-    higher_is_better: false
-metadata:
-  - version: 1.0

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/boolq_ov.yaml

-group: boolq_alt_ov
-task:
-  - boolq_alt_ov_01
-  - boolq_alt_ov_02
-  - boolq_alt_ov_03
-  - boolq_alt_ov_04
-  - boolq_alt_ov_05
-  - boolq_alt_ov_06
-  - boolq_alt_ov_07
-  - boolq_alt_ov_08

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_01/a.yaml

-include: ../_boolq_alt_ov_yaml
-group: boolq_alt_ov_01
-task: boolq_alt_ov_01a
-doc_to_text: !function ../styles.template_01
-doc_to_choice: !function ../styles.choice_01a
-doc_to_decontamination_query: !function ../styles.template_01

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_01/b.yaml

-include: ../_boolq_alt_ov_yaml
-group: boolq_alt_ov_01
-task: boolq_alt_ov_01b
-doc_to_text: !function ../styles.template_01
-doc_to_choice: !function ../styles.choice_01b
-doc_to_decontamination_query: !function ../styles.template_01

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_01/c.yaml

-include: ../_boolq_alt_ov_yaml
-group: boolq_alt_ov_01
-task: boolq_alt_ov_01c
-doc_to_text: !function ../styles.template_01
-doc_to_choice: !function ../styles.choice_01c
-doc_to_decontamination_query: !function ../styles.template_01

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_02/a.yaml

-include: ../_boolq_alt_ov_yaml
-group: boolq_alt_ov_02
-task: boolq_alt_ov_02a
-doc_to_text: !function ../styles.template_02
-doc_to_choice: !function ../styles.choice_02a
-doc_to_decontamination_query: !function ../styles.template_02

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_02/b.yaml

-include: ../_boolq_alt_ov_yaml
-group: boolq_alt_ov_02
-task: boolq_alt_ov_02b
-doc_to_text: !function ../styles.template_02
-doc_to_choice: !function ../styles.choice_02b
-doc_to_decontamination_query: !function ../styles.template_02

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_02/c.yaml

-include: ../_boolq_alt_ov_yaml
-group: boolq_alt_ov_02
-task: boolq_alt_ov_02c
-doc_to_text: !function ../styles.template_02
-doc_to_choice: !function ../styles.choice_02c
-doc_to_decontamination_query: !function ../styles.template_02

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_03/a.yaml

-include: ../_boolq_alt_ov_yaml
-group: boolq_alt_ov_03
-task: boolq_alt_ov_03a
-doc_to_text: !function ../styles.template_03
-doc_to_choice: !function ../styles.choice_03a
-doc_to_decontamination_query: !function ../styles.template_03

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_03/b.yaml

-include: ../_boolq_alt_ov_yaml
-group: boolq_alt_ov_03
-task: boolq_alt_ov_03b
-doc_to_text: !function ../styles.template_03
-doc_to_choice: !function ../styles.choice_03b
-doc_to_decontamination_query: !function ../styles.template_03

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_03/c.yaml

-include: ../_boolq_alt_ov_yaml
-group: boolq_alt_ov_03
-task: boolq_alt_ov_03c
-doc_to_text: !function ../styles.template_03
-doc_to_choice: !function ../styles.choice_03c
-doc_to_decontamination_query: !function ../styles.template_03

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_04/a.yaml

-include: ../_boolq_alt_ov_yaml
-group: boolq_alt_ov_04
-task: boolq_alt_ov_04a
-doc_to_text: !function ../styles.template_04
-doc_to_choice: !function ../styles.choice_04a
-doc_to_decontamination_query: !function ../styles.template_04

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_04/b.yaml

-include: ../_boolq_alt_ov_yaml
-group: boolq_alt_ov_04
-task: boolq_alt_ov_04b
-doc_to_text: !function ../styles.template_04
-doc_to_choice: !function ../styles.choice_04b
-doc_to_decontamination_query: !function ../styles.template_04

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_04/c.yaml

-include: ../_boolq_alt_ov_yaml
-group: boolq_alt_ov_04
-task: boolq_alt_ov_04c
-doc_to_text: !function ../styles.template_04
-doc_to_choice: !function ../styles.choice_04c
-doc_to_decontamination_query: !function ../styles.template_04

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_05/a.yaml

-include: ../_boolq_alt_ov_yaml
-group: boolq_alt_ov_05
-task: boolq_alt_ov_05a
-doc_to_text: !function ../styles.template_05
-doc_to_choice: !function ../styles.choice_05a
-doc_to_decontamination_query: !function ../styles.template_05