Merge remote-tracking branch 'origin/big-refactor' into calibration

lm_eval/tasks/belebele/belebele_tso_Latn.yaml

+"dataset_name": "tso_Latn"
+"include": "_default_template_yaml"
+"task": "belebele_tso_Latn"

lm_eval/tasks/belebele/belebele_tur_Latn.yaml

+"dataset_name": "tur_Latn"
+"include": "_default_template_yaml"
+"task": "belebele_tur_Latn"

lm_eval/tasks/belebele/belebele_ukr_Cyrl.yaml

+"dataset_name": "ukr_Cyrl"
+"include": "_default_template_yaml"
+"task": "belebele_ukr_Cyrl"

lm_eval/tasks/belebele/belebele_urd_Arab.yaml

+"dataset_name": "urd_Arab"
+"include": "_default_template_yaml"
+"task": "belebele_urd_Arab"

lm_eval/tasks/belebele/belebele_urd_Latn.yaml

+"dataset_name": "urd_Latn"
+"include": "_default_template_yaml"
+"task": "belebele_urd_Latn"

lm_eval/tasks/belebele/belebele_uzn_Latn.yaml

+"dataset_name": "uzn_Latn"
+"include": "_default_template_yaml"
+"task": "belebele_uzn_Latn"

lm_eval/tasks/belebele/belebele_vie_Latn.yaml

+"dataset_name": "vie_Latn"
+"include": "_default_template_yaml"
+"task": "belebele_vie_Latn"

lm_eval/tasks/belebele/belebele_war_Latn.yaml

+"dataset_name": "war_Latn"
+"include": "_default_template_yaml"
+"task": "belebele_war_Latn"

lm_eval/tasks/belebele/belebele_wol_Latn.yaml

+"dataset_name": "wol_Latn"
+"include": "_default_template_yaml"
+"task": "belebele_wol_Latn"

lm_eval/tasks/belebele/belebele_xho_Latn.yaml

+"dataset_name": "xho_Latn"
+"include": "_default_template_yaml"
+"task": "belebele_xho_Latn"

lm_eval/tasks/belebele/belebele_yor_Latn.yaml

+"dataset_name": "yor_Latn"
+"include": "_default_template_yaml"
+"task": "belebele_yor_Latn"

lm_eval/tasks/belebele/belebele_zho_Hans.yaml

+"dataset_name": "zho_Hans"
+"include": "_default_template_yaml"
+"task": "belebele_zho_Hans"

lm_eval/tasks/belebele/belebele_zho_Hant.yaml

+"dataset_name": "zho_Hant"
+"include": "_default_template_yaml"
+"task": "belebele_zho_Hant"

lm_eval/tasks/belebele/belebele_zsm_Latn.yaml

+"dataset_name": "zsm_Latn"
+"include": "_default_template_yaml"
+"task": "belebele_zsm_Latn"

lm_eval/tasks/belebele/belebele_zul_Latn.yaml

+"dataset_name": "zul_Latn"
+"include": "_default_template_yaml"
+"task": "belebele_zul_Latn"

lm_eval/tasks/benchmarks/flan/flan_anli.yaml

+group: flan_anli
+task:
+  - include: yaml_templates/held_in_template_yaml
+    task: anli_r1
+    dataset_path: anli
+    use_prompt: prompt_templates/anli.yaml:*
+    validation_split: dev_r1
+  - include: yaml_templates/held_in_template_yaml
+    task: anli_r2
+    dataset_path: anli
+    use_prompt: prompt_templates/anli.yaml:*
+    validation_split: dev_r2
+  - include: yaml_templates/held_in_template_yaml
+    task: anli_r3
+    dataset_path: anli
+    use_prompt: prompt_templates/anli.yaml:*
+    validation_split: dev_r3

lm_eval/tasks/benchmarks/flan/flan_arc.yaml

+group: flan_arc
+task:
+  - include: yaml_templates/held_in_template_yaml
+    task: arc_easy
+    dataset_path: ai2_arc
+    dataset_name: ARC-Easy
+    use_prompt: prompt_templates/arc.yaml:*
+    validation_split: validation
+  - include: yaml_templates/held_in_template_yaml
+    task: arc_challenge
+    dataset_path: ai2_arc
+    dataset_name: ARC-Challenge
+    use_prompt: prompt_templates/arc.yaml:*
+    validation_split: validation

lm_eval/tasks/benchmarks/flan/flan_boolq.yaml

+group: flan_boolq
+task:
+  - include: yaml_templates/held_in_template_yaml
+    dataset_path: super_glue
+    dataset_name: boolq
+    use_prompt: prompt_templates/boolq.yaml:*
+    validation_split: validation

lm_eval/tasks/benchmarks/flan/flan_cot.yaml

+group: flan_cot
+task:
+  - include: yaml_templates/cot_template_yaml
+    dataset_path: gsmk
+    dataset_name: boolq
+    use_prompt: promptsource:*
+    validation_split: validation
+  - include: yaml_templates/cot_template_yaml
+    dataset_path: EleutherAI/asdiv
+    use_prompt: promptsource:*
+    validation_split: validation

lm_eval/tasks/benchmarks/flan/flan_held_in.yaml

+group: flan_held_in
+task:
+  - flan_boolq
+  - flan_rte
+  - flan_anli
+  - flan_arc