Merge branch 'big-refactor' into model-written-eval

lm_eval/tasks/belebele/belebele_uzn_Latn.yaml

+"dataset_name": "uzn_Latn"
+"include": "_default_template_yaml"
+"task": "belebele_uzn_Latn"

lm_eval/tasks/belebele/belebele_vie_Latn.yaml

+"dataset_name": "vie_Latn"
+"include": "_default_template_yaml"
+"task": "belebele_vie_Latn"

lm_eval/tasks/belebele/belebele_war_Latn.yaml

+"dataset_name": "war_Latn"
+"include": "_default_template_yaml"
+"task": "belebele_war_Latn"

lm_eval/tasks/belebele/belebele_wol_Latn.yaml

+"dataset_name": "wol_Latn"
+"include": "_default_template_yaml"
+"task": "belebele_wol_Latn"

lm_eval/tasks/belebele/belebele_xho_Latn.yaml

+"dataset_name": "xho_Latn"
+"include": "_default_template_yaml"
+"task": "belebele_xho_Latn"

lm_eval/tasks/belebele/belebele_yor_Latn.yaml

+"dataset_name": "yor_Latn"
+"include": "_default_template_yaml"
+"task": "belebele_yor_Latn"

lm_eval/tasks/belebele/belebele_zho_Hans.yaml

+"dataset_name": "zho_Hans"
+"include": "_default_template_yaml"
+"task": "belebele_zho_Hans"

lm_eval/tasks/belebele/belebele_zho_Hant.yaml

+"dataset_name": "zho_Hant"
+"include": "_default_template_yaml"
+"task": "belebele_zho_Hant"

lm_eval/tasks/belebele/belebele_zsm_Latn.yaml

+"dataset_name": "zsm_Latn"
+"include": "_default_template_yaml"
+"task": "belebele_zsm_Latn"

lm_eval/tasks/belebele/belebele_zul_Latn.yaml

+"dataset_name": "zul_Latn"
+"include": "_default_template_yaml"
+"task": "belebele_zul_Latn"

lm_eval/tasks/benchmarks/flan/yaml_templates/cot_template_yaml

 group: flan-cot
-output_type: greedy_until
+output_type: generate_until
 validation_split: validation
 doc_to_target: "{{answer}}"
 metric_list:

lm_eval/tasks/benchmarks/flan/yaml_templates/held_in_template_yaml

-output_type: greedy_until
+output_type: generate_until
 validation_split: validation
 metric_list:
   - metric: exact_match

lm_eval/tasks/benchmarks/t0_eval.yaml

@@ -6,7 +6,7 @@ task:
     use_prompt: promptsource:*
     training_split: train
     validation_split: validation
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean
@@ -19,7 +19,7 @@ task:
     use_prompt: promptsource:*
     training_split: train
     validation_split: validation
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean
@@ -32,7 +32,7 @@ task:
     use_prompt: promptsource:*
     training_split: train
     validation_split: validation
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean
@@ -44,7 +44,7 @@ task:
     use_prompt: promptsource:*
     training_split: train
     validation_split: validation
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean
@@ -56,7 +56,7 @@ task:
     use_prompt: promptsource:*
     training_split: train_r1
     validation_split: dev_r1
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean
@@ -68,7 +68,7 @@ task:
     use_prompt: promptsource:*
     training_split: train_r2
     validation_split: dev_r2
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean
@@ -80,7 +80,7 @@ task:
     use_prompt: promptsource:*
     training_split: train_r3
     validation_split: dev_r3
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean
@@ -93,7 +93,7 @@ task:
     use_prompt: promptsource:*
     training_split: train
     validation_split: validation
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean
@@ -105,7 +105,7 @@ task:
     use_prompt: promptsource:*
     training_split: train
     validation_split: validation
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean
@@ -118,7 +118,7 @@ task:
     use_prompt: promptsource:*
     training_split: train
     validation_split: validation
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean

lm_eval/tasks/bigbench/generate_tasks.py

@@ -175,8 +175,8 @@ all_subtasks = [
 def main() -> None:
 
     for path, task_type in zip(
-        ["multiple_choice", "greedy_until"],
-        ["multiple_choice_template_yaml", "greedy_until_template_yaml"],
+        ["multiple_choice", "generate_until"],
+        ["multiple_choice_template_yaml", "generate_until_template_yaml"],
     ):
         os.makedirs(path, exist_ok=True)
         for task in all_subtasks:

lm_eval/tasks/bigbench/greedy_until/abstract_narrative_understanding.yaml → lm_eval/tasks/bigbench/generate_until/abstract_narrative_understanding.yaml

 # Generated by utils.py
 dataset_name: abstract_narrative_understanding_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_abstract_narrative_understanding_greedy_until
+include: ../generate_until_template_yaml
+task: bigbench_abstract_narrative_understanding_generate_until

lm_eval/tasks/bigbench/greedy_until/anachronisms.yaml → lm_eval/tasks/bigbench/generate_until/anachronisms.yaml

 # Generated by utils.py
 dataset_name: anachronisms_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_anachronisms_greedy_until
+include: ../generate_until_template_yaml
+task: bigbench_anachronisms_generate_until

lm_eval/tasks/bigbench/greedy_until/analogical_similarity.yaml → lm_eval/tasks/bigbench/generate_until/analogical_similarity.yaml

 # Generated by utils.py
 dataset_name: analogical_similarity_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_analogical_similarity_greedy_until
+include: ../generate_until_template_yaml
+task: bigbench_analogical_similarity_generate_until

lm_eval/tasks/bigbench/greedy_until/analytic_entailment.yaml → lm_eval/tasks/bigbench/generate_until/analytic_entailment.yaml

 # Generated by utils.py
 dataset_name: analytic_entailment_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_analytic_entailment_greedy_until
+include: ../generate_until_template_yaml
+task: bigbench_analytic_entailment_generate_until

lm_eval/tasks/bigbench/greedy_until/arithmetic.yaml → lm_eval/tasks/bigbench/generate_until/arithmetic.yaml

 # Generated by utils.py
 dataset_name: arithmetic_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_arithmetic_greedy_until
+include: ../generate_until_template_yaml
+task: bigbench_arithmetic_generate_until