Merge pull request #927 from EleutherAI/change_output_type

[Refactor] change all mentions of `greedy_until` to `generate_until`

lm_eval/tasks/translation/utils.py

@@ -58,7 +58,7 @@ def gen_lang_yamls(output_dir: str, overwrite: bool) -> None:
                 try:
                     source, target = code_to_language(src), code_to_language(tgt)
 
-                    groups = ["greedy_until", "translation", lang]
+                    groups = ["generate_until", "translation", lang]
                     if lang in gpt3_translation_benchmarks.keys():
                         groups += ["gpt3_translation_benchmarks"]
 

lm_eval/tasks/translation/wmt14_en-fr.yaml

@@ -6,7 +6,7 @@ doc_to_text: 'English phrase: {{translation["en"]}}
 
   French phrase:'
 group:
-- greedy_until
+- generate_until
 - translation
 - wmt14
 - gpt3_translation_benchmarks

lm_eval/tasks/translation/wmt14_fr-en.yaml

@@ -6,7 +6,7 @@ doc_to_text: 'French phrase: {{translation["fr"]}}
 
   English phrase:'
 group:
-- greedy_until
+- generate_until
 - translation
 - wmt14
 - gpt3_translation_benchmarks

lm_eval/tasks/translation/wmt16_de-en.yaml

@@ -6,7 +6,7 @@ doc_to_text: 'German phrase: {{translation["de"]}}
 
   English phrase:'
 group:
-- greedy_until
+- generate_until
 - translation
 - wmt16
 - gpt3_translation_benchmarks

lm_eval/tasks/translation/wmt16_en-de.yaml

@@ -6,7 +6,7 @@ doc_to_text: 'English phrase: {{translation["en"]}}
 
   German phrase:'
 group:
-- greedy_until
+- generate_until
 - translation
 - wmt16
 - gpt3_translation_benchmarks

lm_eval/tasks/translation/wmt16_en-ro.yaml

@@ -6,7 +6,7 @@ doc_to_text: 'English phrase: {{translation["en"]}}
 
   Romanian phrase:'
 group:
-- greedy_until
+- generate_until
 - translation
 - wmt16
 - gpt3_translation_benchmarks

lm_eval/tasks/translation/wmt16_ro-en.yaml

@@ -6,7 +6,7 @@ doc_to_text: 'Romanian phrase: {{translation["ro"]}}
 
   English phrase:'
 group:
-- greedy_until
+- generate_until
 - translation
 - wmt16
 - gpt3_translation_benchmarks

lm_eval/tasks/translation/wmt_common_yaml

-output_type: greedy_until
+output_type: generate_until
 training_split: train
 validation_split: validation
 fewshot_split: validation

lm_eval/tasks/triviaqa/default.yaml

 task: triviaqa
 dataset_path: trivia_qa
 dataset_name: rc.nocontext
-output_type: greedy_until
+output_type: generate_until
 training_split: train
 validation_split: validation
 doc_to_text: "Question: {{question}}?\nAnswer:"

lm_eval/tasks/truthfulqa/truthfulqa_gen.yaml

@@ -3,7 +3,7 @@ group:
 task: truthfulqa_gen
 dataset_path: truthful_qa
 dataset_name: generation
-output_type: greedy_until
+output_type: generate_until
 training_split: null
 validation_split: validation
 test_split: null

lm_eval/tasks/unscramble/anagrams1.yaml

@@ -3,7 +3,7 @@ group:
 task: anagrams1
 dataset_path: EleutherAI/unscramble
 dataset_name: mid_word_1_anagrams
-output_type: greedy_until
+output_type: generate_until
 test_split: validation
 doc_to_text: "{{context}}"
 doc_to_target: "{{completion}}"

lm_eval/tasks/unscramble/anagrams2.yaml

@@ -3,7 +3,7 @@ group:
 task: anagrams2
 dataset_path: EleutherAI/unscramble
 dataset_name: mid_word_2_anagrams
-output_type: greedy_until
+output_type: generate_until
 test_split: validation
 doc_to_text: "{{context}}"
 doc_to_target: "{{completion}}"

lm_eval/tasks/unscramble/cycle_letters.yaml

@@ -3,7 +3,7 @@ group:
 task: cycle_letters
 dataset_path: EleutherAI/unscramble
 dataset_name: cycle_letters_in_word
-output_type: greedy_until
+output_type: generate_until
 test_split: validation
 doc_to_text: "{{context}}"
 doc_to_target: "{{completion}}"

lm_eval/tasks/unscramble/random_insertion.yaml

@@ -3,7 +3,7 @@ group:
 task: random_insertion
 dataset_path: EleutherAI/unscramble
 dataset_name: random_insertion_in_word
-output_type: greedy_until
+output_type: generate_until
 test_split: validation
 doc_to_text: "{{context}}"
 doc_to_target: "{{completion}}"

lm_eval/tasks/unscramble/reversed_words.yaml

@@ -3,7 +3,7 @@ group:
 task: reversed_words
 dataset_path: EleutherAI/unscramble
 dataset_name: reversed_words
-output_type: greedy_until
+output_type: generate_until
 test_split: validation
 doc_to_text: "{{context}}"
 doc_to_target: "{{completion}}"

lm_eval/tasks/wmt2016/ro_en-t5_prompt.yaml

@@ -5,7 +5,7 @@ dataset_path: wmt16
 dataset_name: ro-en
 training_split: train
 validation_split: validation
-output_type: greedy_until
+output_type: generate_until
 doc_to_text: "translate English to Romanian: {{translation.en}}"
 doc_to_target: "{{translation.ro}}"
 metric_list: