Merge branch 'big-refactor' of...

Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evaluation-harness into mmlu_subgroups

lm_eval/tasks/benchmarks/t0_eval.yaml

@@ -6,7 +6,7 @@ task:
     use_prompt: promptsource:*
     training_split: train
     validation_split: validation
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean
@@ -19,7 +19,7 @@ task:
     use_prompt: promptsource:*
     training_split: train
     validation_split: validation
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean
@@ -32,7 +32,7 @@ task:
     use_prompt: promptsource:*
     training_split: train
     validation_split: validation
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean
@@ -44,7 +44,7 @@ task:
     use_prompt: promptsource:*
     training_split: train
     validation_split: validation
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean
@@ -56,7 +56,7 @@ task:
     use_prompt: promptsource:*
     training_split: train_r1
     validation_split: dev_r1
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean
@@ -68,7 +68,7 @@ task:
     use_prompt: promptsource:*
     training_split: train_r2
     validation_split: dev_r2
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean
@@ -80,7 +80,7 @@ task:
     use_prompt: promptsource:*
     training_split: train_r3
     validation_split: dev_r3
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean
@@ -93,7 +93,7 @@ task:
     use_prompt: promptsource:*
     training_split: train
     validation_split: validation
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean
@@ -105,7 +105,7 @@ task:
     use_prompt: promptsource:*
     training_split: train
     validation_split: validation
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean
@@ -118,7 +118,7 @@ task:
     use_prompt: promptsource:*
     training_split: train
     validation_split: validation
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean

lm_eval/tasks/bigbench/generate_tasks.py

@@ -175,8 +175,8 @@ all_subtasks = [
 def main() -> None:
 
     for path, task_type in zip(
-        ["multiple_choice", "greedy_until"],
-        ["multiple_choice_template_yaml", "greedy_until_template_yaml"],
+        ["multiple_choice", "generate_until"],
+        ["multiple_choice_template_yaml", "generate_until_template_yaml"],
     ):
         os.makedirs(path, exist_ok=True)
         for task in all_subtasks:

lm_eval/tasks/bigbench/greedy_until/abstract_narrative_understanding.yaml → lm_eval/tasks/bigbench/generate_until/abstract_narrative_understanding.yaml

 # Generated by utils.py
 dataset_name: abstract_narrative_understanding_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_abstract_narrative_understanding_greedy_until
+include: ../generate_until_template_yaml
+task: bigbench_abstract_narrative_understanding_generate_until

lm_eval/tasks/bigbench/greedy_until/anachronisms.yaml → lm_eval/tasks/bigbench/generate_until/anachronisms.yaml

 # Generated by utils.py
 dataset_name: anachronisms_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_anachronisms_greedy_until
+include: ../generate_until_template_yaml
+task: bigbench_anachronisms_generate_until

lm_eval/tasks/bigbench/greedy_until/analogical_similarity.yaml → lm_eval/tasks/bigbench/generate_until/analogical_similarity.yaml

 # Generated by utils.py
 dataset_name: analogical_similarity_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_analogical_similarity_greedy_until
+include: ../generate_until_template_yaml
+task: bigbench_analogical_similarity_generate_until

lm_eval/tasks/bigbench/greedy_until/analytic_entailment.yaml → lm_eval/tasks/bigbench/generate_until/analytic_entailment.yaml

 # Generated by utils.py
 dataset_name: analytic_entailment_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_analytic_entailment_greedy_until
+include: ../generate_until_template_yaml
+task: bigbench_analytic_entailment_generate_until

lm_eval/tasks/bigbench/greedy_until/arithmetic.yaml → lm_eval/tasks/bigbench/generate_until/arithmetic.yaml

 # Generated by utils.py
 dataset_name: arithmetic_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_arithmetic_greedy_until
+include: ../generate_until_template_yaml
+task: bigbench_arithmetic_generate_until

lm_eval/tasks/bigbench/greedy_until/ascii_word_recognition.yaml → lm_eval/tasks/bigbench/generate_until/ascii_word_recognition.yaml

 # Generated by utils.py
 dataset_name: ascii_word_recognition_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_ascii_word_recognition_greedy_until
+include: ../generate_until_template_yaml
+task: bigbench_ascii_word_recognition_generate_until

lm_eval/tasks/bigbench/greedy_until/authorship_verification.yaml → lm_eval/tasks/bigbench/generate_until/authorship_verification.yaml

 # Generated by utils.py
 dataset_name: authorship_verification_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_authorship_verification_greedy_until
+include: ../generate_until_template_yaml
+task: bigbench_authorship_verification_generate_until

lm_eval/tasks/bigbench/greedy_until/auto_categorization.yaml → lm_eval/tasks/bigbench/generate_until/auto_categorization.yaml

 # Generated by utils.py
 dataset_name: auto_categorization_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_auto_categorization_greedy_until
+include: ../generate_until_template_yaml
+task: bigbench_auto_categorization_generate_until

lm_eval/tasks/bigbench/greedy_until/auto_debugging.yaml → lm_eval/tasks/bigbench/generate_until/auto_debugging.yaml

 # Generated by utils.py
 dataset_name: auto_debugging_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_auto_debugging_greedy_until
+include: ../generate_until_template_yaml
+task: bigbench_auto_debugging_generate_until

lm_eval/tasks/bigbench/greedy_until/bbq_lite_json.yaml → lm_eval/tasks/bigbench/generate_until/bbq_lite_json.yaml

 # Generated by utils.py
 dataset_name: bbq_lite_json_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_bbq_lite_json_greedy_until
+include: ../generate_until_template_yaml
+task: bigbench_bbq_lite_json_generate_until

lm_eval/tasks/bigbench/greedy_until/bridging_anaphora_resolution_barqa.yaml → lm_eval/tasks/bigbench/generate_until/bridging_anaphora_resolution_barqa.yaml

 # Generated by utils.py
 dataset_name: bridging_anaphora_resolution_barqa_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_bridging_anaphora_resolution_barqa_greedy_until
+include: ../generate_until_template_yaml
+task: bigbench_bridging_anaphora_resolution_barqa_generate_until

lm_eval/tasks/bigbench/greedy_until/causal_judgment.yaml → lm_eval/tasks/bigbench/generate_until/causal_judgment.yaml

 # Generated by utils.py
 dataset_name: causal_judgment_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_causal_judgment_greedy_until
+include: ../generate_until_template_yaml
+task: bigbench_causal_judgment_generate_until

lm_eval/tasks/bigbench/greedy_until/cause_and_effect.yaml → lm_eval/tasks/bigbench/generate_until/cause_and_effect.yaml

 # Generated by utils.py
 dataset_name: cause_and_effect_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_cause_and_effect_greedy_until
+include: ../generate_until_template_yaml
+task: bigbench_cause_and_effect_generate_until

lm_eval/tasks/bigbench/greedy_until/checkmate_in_one.yaml → lm_eval/tasks/bigbench/generate_until/checkmate_in_one.yaml

 # Generated by utils.py
 dataset_name: checkmate_in_one_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_checkmate_in_one_greedy_until
+include: ../generate_until_template_yaml
+task: bigbench_checkmate_in_one_generate_until

lm_eval/tasks/bigbench/greedy_until/chess_state_tracking.yaml → lm_eval/tasks/bigbench/generate_until/chess_state_tracking.yaml

 # Generated by utils.py
 dataset_name: chess_state_tracking_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_chess_state_tracking_greedy_until
+include: ../generate_until_template_yaml
+task: bigbench_chess_state_tracking_generate_until

lm_eval/tasks/bigbench/greedy_until/chinese_remainder_theorem.yaml → lm_eval/tasks/bigbench/generate_until/chinese_remainder_theorem.yaml

 # Generated by utils.py
 dataset_name: chinese_remainder_theorem_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_chinese_remainder_theorem_greedy_until
+include: ../generate_until_template_yaml
+task: bigbench_chinese_remainder_theorem_generate_until

lm_eval/tasks/bigbench/greedy_until/cifar10_classification.yaml → lm_eval/tasks/bigbench/generate_until/cifar10_classification.yaml

 # Generated by utils.py
 dataset_name: cifar10_classification_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_cifar10_classification_greedy_until
+include: ../generate_until_template_yaml
+task: bigbench_cifar10_classification_generate_until

lm_eval/tasks/bigbench/greedy_until/code_line_description.yaml → lm_eval/tasks/bigbench/generate_until/code_line_description.yaml

 # Generated by utils.py
 dataset_name: code_line_description_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_code_line_description_greedy_until
+include: ../generate_until_template_yaml
+task: bigbench_code_line_description_generate_until