Merge branch 'big-refactor' into fix-unittests

lm_eval/tasks/translation/wmt_common_yaml

-output_type: greedy_until
+output_type: generate_until
 training_split: train
 validation_split: validation
 fewshot_split: validation

lm_eval/tasks/triviaqa/default.yaml

 task: triviaqa
 dataset_path: trivia_qa
 dataset_name: rc.nocontext
-output_type: greedy_until
+output_type: generate_until
 training_split: train
 validation_split: validation
 doc_to_text: "Question: {{question}}?\nAnswer:"

lm_eval/tasks/truthfulqa/truthfulqa_gen.yaml

@@ -3,7 +3,7 @@ group:
 task: truthfulqa_gen
 dataset_path: truthful_qa
 dataset_name: generation
-output_type: greedy_until
+output_type: generate_until
 training_split: null
 validation_split: validation
 test_split: null

lm_eval/tasks/unscramble/anagrams1.yaml

@@ -3,7 +3,7 @@ group:
 task: anagrams1
 dataset_path: EleutherAI/unscramble
 dataset_name: mid_word_1_anagrams
-output_type: greedy_until
+output_type: generate_until
 test_split: validation
 doc_to_text: "{{context}}"
 doc_to_target: "{{completion}}"

lm_eval/tasks/unscramble/anagrams2.yaml

@@ -3,7 +3,7 @@ group:
 task: anagrams2
 dataset_path: EleutherAI/unscramble
 dataset_name: mid_word_2_anagrams
-output_type: greedy_until
+output_type: generate_until
 test_split: validation
 doc_to_text: "{{context}}"
 doc_to_target: "{{completion}}"

lm_eval/tasks/unscramble/cycle_letters.yaml

@@ -3,7 +3,7 @@ group:
 task: cycle_letters
 dataset_path: EleutherAI/unscramble
 dataset_name: cycle_letters_in_word
-output_type: greedy_until
+output_type: generate_until
 test_split: validation
 doc_to_text: "{{context}}"
 doc_to_target: "{{completion}}"

lm_eval/tasks/unscramble/random_insertion.yaml

@@ -3,7 +3,7 @@ group:
 task: random_insertion
 dataset_path: EleutherAI/unscramble
 dataset_name: random_insertion_in_word
-output_type: greedy_until
+output_type: generate_until
 test_split: validation
 doc_to_text: "{{context}}"
 doc_to_target: "{{completion}}"

lm_eval/tasks/unscramble/reversed_words.yaml

@@ -3,7 +3,7 @@ group:
 task: reversed_words
 dataset_path: EleutherAI/unscramble
 dataset_name: reversed_words
-output_type: greedy_until
+output_type: generate_until
 test_split: validation
 doc_to_text: "{{context}}"
 doc_to_target: "{{completion}}"

lm_eval/tasks/wmt2016/ro_en-t5_prompt.yaml

@@ -5,7 +5,7 @@ dataset_path: wmt16
 dataset_name: ro-en
 training_split: train
 validation_split: validation
-output_type: greedy_until
+output_type: generate_until
 doc_to_text: "translate English to Romanian: {{translation.en}}"
 doc_to_target: "{{translation.ro}}"
 metric_list:

lm_eval/utils.py

@@ -78,7 +78,7 @@ def chunks(iter, n: int = 0, fn=None):
     arr = []
     for i, x in enumerate(iter):
         arr.append(x)
-        if len(arr) == (fn(i) if fn else n):
+        if len(arr) == (fn(i, iter) if fn else n):
             yield arr
             arr = []