Merge branch 'cont-metrics' of https://github.com/EleutherAI/lm-evaluation-harness into alt_worlds

lm_eval/tasks/bigbench/greedy_until/tense.yaml

-# Generated by utils.py
-dataset_name: tense_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_tense_greedy_until

lm_eval/tasks/bigbench/greedy_until/timedial.yaml

-# Generated by utils.py
-dataset_name: timedial_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_timedial_greedy_until

lm_eval/tasks/bigbench/greedy_until/topical_chat.yaml

-# Generated by utils.py
-dataset_name: topical_chat_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_topical_chat_greedy_until

lm_eval/tasks/bigbench/greedy_until/tracking_shuffled_objects.yaml

-# Generated by utils.py
-dataset_name: tracking_shuffled_objects_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_tracking_shuffled_objects_greedy_until

lm_eval/tasks/bigbench/greedy_until/understanding_fables.yaml

-# Generated by utils.py
-dataset_name: understanding_fables_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_understanding_fables_greedy_until

lm_eval/tasks/bigbench/greedy_until/undo_permutation.yaml

-# Generated by utils.py
-dataset_name: undo_permutation_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_undo_permutation_greedy_until

lm_eval/tasks/bigbench/greedy_until/unit_conversion.yaml

-# Generated by utils.py
-dataset_name: unit_conversion_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_unit_conversion_greedy_until

lm_eval/tasks/bigbench/greedy_until/unit_interpretation.yaml

-# Generated by utils.py
-dataset_name: unit_interpretation_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_unit_interpretation_greedy_until

lm_eval/tasks/bigbench/greedy_until/unnatural_in_context_learning.yaml

-# Generated by utils.py
-dataset_name: unnatural_in_context_learning_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_unnatural_in_context_learning_greedy_until

lm_eval/tasks/bigbench/greedy_until/vitaminc_fact_verification.yaml

-# Generated by utils.py
-dataset_name: vitaminc_fact_verification_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_vitaminc_fact_verification_greedy_until

lm_eval/tasks/bigbench/greedy_until/what_is_the_tao.yaml

-# Generated by utils.py
-dataset_name: what_is_the_tao_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_what_is_the_tao_greedy_until

lm_eval/tasks/bigbench/greedy_until/which_wiki_edit.yaml

-# Generated by utils.py
-dataset_name: which_wiki_edit_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_which_wiki_edit_greedy_until

lm_eval/tasks/bigbench/greedy_until/winowhy.yaml

-# Generated by utils.py
-dataset_name: winowhy_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_winowhy_greedy_until

lm_eval/tasks/bigbench/greedy_until/word_sorting.yaml

-# Generated by utils.py
-dataset_name: word_sorting_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_word_sorting_greedy_until

lm_eval/tasks/bigbench/greedy_until/word_unscrambling.yaml

-# Generated by utils.py
-dataset_name: word_unscrambling_zero_shot
-include: ../greedy_until_template_yaml
-task: bigbench_word_unscrambling_greedy_until

lm_eval/tasks/bigbench/greedy_until_template_yaml

-group: bigbench
-dataset_path: bigbench # will switch to `hails/bigbench` when all tasks are pushed
-output_type: greedy_until
-dataset_kwargs:
-  # num_shots: 0 # TODO: num of shots for `bigbench` HF dataset should be controlled through this, not through the typical methods
-  # subtask_name: null
-test_split: default
-doc_to_text: inputs
-doc_to_target: "{{targets[0]}}"
-generation_kwargs:
-  max_length: 128
-metric_list:
-  - metric: exact_match
-    aggregation: mean
-    higher_is_better: true
-    ignore_punctuation: true

lm_eval/tasks/code_x_glue/code-text/go.yaml

@@ -5,7 +5,7 @@ dataset_path: CM/codexglue_code2text_go
 training_split: train
 validation_split: validation
 test_split: test
-output_type: greedy_until
+output_type: generate_until
 generation_kwargs:
   num_beams: 10
   max_length: 128

lm_eval/tasks/code_x_glue/code-text/java.yaml

@@ -5,7 +5,7 @@ dataset_path: CM/codexglue_code2text_java
 training_split: train
 validation_split: validation
 test_split: test
-output_type: greedy_until
+output_type: generate_until
 generation_kwargs:
   num_beams: 10
   max_length: 128

lm_eval/tasks/code_x_glue/code-text/javascript.yaml

@@ -5,7 +5,7 @@ dataset_path: CM/codexglue_code2text_javascript
 training_split: train
 validation_split: validation
 test_split: test
-output_type: greedy_until
+output_type: generate_until
 generation_kwargs:
   num_beams: 10
   max_length: 128

lm_eval/tasks/code_x_glue/code-text/php.yaml

@@ -5,7 +5,7 @@ dataset_path: CM/codexglue_code2text_php
 training_split: train
 validation_split: validation
 test_split: test
-output_type: greedy_until
+output_type: generate_until
 generation_kwargs:
   num_beams: 10
   max_length: 128