Merge branch 'cont-metrics' of https://github.com/EleutherAI/lm-evaluation-harness into alt_worlds

lm_eval/tasks/belebele/belebele_ukr_Cyrl.yaml

+"dataset_name": "ukr_Cyrl"
+"include": "_default_template_yaml"
+"task": "belebele_ukr_Cyrl"

lm_eval/tasks/belebele/belebele_urd_Arab.yaml

+"dataset_name": "urd_Arab"
+"include": "_default_template_yaml"
+"task": "belebele_urd_Arab"

lm_eval/tasks/belebele/belebele_urd_Latn.yaml

+"dataset_name": "urd_Latn"
+"include": "_default_template_yaml"
+"task": "belebele_urd_Latn"

lm_eval/tasks/belebele/belebele_uzn_Latn.yaml

+"dataset_name": "uzn_Latn"
+"include": "_default_template_yaml"
+"task": "belebele_uzn_Latn"

lm_eval/tasks/belebele/belebele_vie_Latn.yaml

+"dataset_name": "vie_Latn"
+"include": "_default_template_yaml"
+"task": "belebele_vie_Latn"

lm_eval/tasks/belebele/belebele_war_Latn.yaml

+"dataset_name": "war_Latn"
+"include": "_default_template_yaml"
+"task": "belebele_war_Latn"

lm_eval/tasks/belebele/belebele_wol_Latn.yaml

+"dataset_name": "wol_Latn"
+"include": "_default_template_yaml"
+"task": "belebele_wol_Latn"

lm_eval/tasks/belebele/belebele_xho_Latn.yaml

+"dataset_name": "xho_Latn"
+"include": "_default_template_yaml"
+"task": "belebele_xho_Latn"

lm_eval/tasks/belebele/belebele_yor_Latn.yaml

+"dataset_name": "yor_Latn"
+"include": "_default_template_yaml"
+"task": "belebele_yor_Latn"

lm_eval/tasks/belebele/belebele_zho_Hans.yaml

+"dataset_name": "zho_Hans"
+"include": "_default_template_yaml"
+"task": "belebele_zho_Hans"

lm_eval/tasks/belebele/belebele_zho_Hant.yaml

+"dataset_name": "zho_Hant"
+"include": "_default_template_yaml"
+"task": "belebele_zho_Hant"

lm_eval/tasks/belebele/belebele_zsm_Latn.yaml

+"dataset_name": "zsm_Latn"
+"include": "_default_template_yaml"
+"task": "belebele_zsm_Latn"

lm_eval/tasks/belebele/belebele_zul_Latn.yaml

+"dataset_name": "zul_Latn"
+"include": "_default_template_yaml"
+"task": "belebele_zul_Latn"

lm_eval/tasks/benchmarks/flan/yaml_templates/cot_template_yaml

 group: flan-cot
-output_type: greedy_until
+output_type: generate_until
 validation_split: validation
 doc_to_target: "{{answer}}"
 metric_list:

lm_eval/tasks/benchmarks/flan/yaml_templates/held_in_template_yaml

-output_type: greedy_until
+output_type: generate_until
 validation_split: validation
 metric_list:
   - metric: exact_match

lm_eval/tasks/benchmarks/pythia.yaml

@@ -9,4 +9,4 @@ task:
   - wsc
   - ai2_arc
   - blimp
-  - hendrycksTest*
+  - mmlu

lm_eval/tasks/benchmarks/t0_eval.yaml

@@ -6,7 +6,7 @@ task:
     use_prompt: promptsource:*
     training_split: train
     validation_split: validation
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean
@@ -19,7 +19,7 @@ task:
     use_prompt: promptsource:*
     training_split: train
     validation_split: validation
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean
@@ -32,7 +32,7 @@ task:
     use_prompt: promptsource:*
     training_split: train
     validation_split: validation
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean
@@ -44,7 +44,7 @@ task:
     use_prompt: promptsource:*
     training_split: train
     validation_split: validation
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean
@@ -56,7 +56,7 @@ task:
     use_prompt: promptsource:*
     training_split: train_r1
     validation_split: dev_r1
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean
@@ -68,7 +68,7 @@ task:
     use_prompt: promptsource:*
     training_split: train_r2
     validation_split: dev_r2
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean
@@ -80,7 +80,7 @@ task:
     use_prompt: promptsource:*
     training_split: train_r3
     validation_split: dev_r3
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean
@@ -93,7 +93,7 @@ task:
     use_prompt: promptsource:*
     training_split: train
     validation_split: validation
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean
@@ -105,7 +105,7 @@ task:
     use_prompt: promptsource:*
     training_split: train
     validation_split: validation
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean
@@ -118,7 +118,7 @@ task:
     use_prompt: promptsource:*
     training_split: train
     validation_split: validation
-    output_type: greedy_until
+    output_type: generate_until
     metric_list:
       - metric: exact_match
         aggregation: mean

lm_eval/tasks/bigbench/aux_metric.py

+from textdistance import levenshtein
+from transformers import AutoTokenizer
+
+# Change this tokenizer to fit with the model you are using.
+tokenizer = AutoTokenizer.from_pretrained("EleutherAI/pythia-2.8b", max_new_tokens=128)
+
+
+def token_edit_distance(references, predictions, **kwargs):
+    print(references)
+    print(predictions)
+    print("###")
+    ref_tokens = tokenizer.encode(references[0])
+    pred_tokens = tokenizer.encode(predictions[0])
+    return levenshtein.distance(ref_tokens, pred_tokens)

lm_eval/tasks/bigbench/generate_tasks.py

@@ -175,8 +175,8 @@ all_subtasks = [
 def main() -> None:
 
     for path, task_type in zip(
-        ["multiple_choice", "greedy_until"],
-        ["multiple_choice_template_yaml", "greedy_until_template_yaml"],
+        ["multiple_choice", "generate_until"],
+        ["multiple_choice_template_yaml", "generate_until_template_yaml"],
     ):
         os.makedirs(path, exist_ok=True)
         for task in all_subtasks: