edit stop token

lm_eval/tasks/super_glue/boolq/t5-prompt.yaml

@@ -9,6 +9,9 @@ output_type: greedy_until
 doc_to_text: "boolq passage: {{passage}} question: {{question}}"
 doc_to_target: label
 doc_to_choice: ['False', 'True']
+generation_kwargs:
+  until:
+    - "</s>"
 metric_list:
   - metric: exact_match
     aggregation: mean

lm_eval/tasks/super_glue/cb/t5-prompt.yaml

@@ -9,6 +9,9 @@ output_type: greedy_until
 doc_to_text: "cb hypothesis: {{hypothesis}} premise: {{premise}}"
 doc_to_target: label
 doc_to_choice: ['entailment', 'contradiction', 'neutral']
+generation_kwargs:
+  until:
+    - "</s>"
 metric_list:
   - metric: exact_match
     aggregation: mean

lm_eval/tasks/super_glue/cb/t5_utils.py

@@ -4,7 +4,7 @@ import sklearn.metrics
 def mean_3class_f1(predictions, references):  # This is a passthrough function
 
     string_label = ["entailment", "contradiction", "neutral"]
-    predictions = string_label.index(predictions[0])
+    predictions = string_label.index(predictions[0]) if predictions[0] in string_label else 0
     references = string_label.index(references[0])
 
     return (predictions, references)

lm_eval/tasks/super_glue/copa/t5-prompt.yaml

@@ -9,6 +9,9 @@ output_type: greedy_until
 doc_to_text: "copa choice1: {{choice1}} choice2: {{choice2}} premise: {{premise}} question: {{question}}"
 doc_to_target: label
 doc_to_choice: ['choice1', 'choice2']
+generation_kwargs:
+  until:
+    - "</s>"
 metric_list:
   - metric: exact_match
     aggregation: mean

lm_eval/tasks/super_glue/multirc/t5-prompt.yaml

@@ -12,8 +12,6 @@ doc_to_choice: "{% set group_id = idx.question|string %}{{[group_id+'_False', gr
 generation_kwargs:
   until:
     - "</s>"
-  do_sample: false
-  temperature: 0.5
 metric_list:
   - metric: !function t5_utils.f1
     aggregation: !function t5_utils.agg_f1

lm_eval/tasks/super_glue/record/t5-prompt.yaml

@@ -8,6 +8,9 @@ output_type: greedy_until
 process_docs: !function t5_utils.process_docs
 doc_to_text: !function t5_utils.doc_to_text
 doc_to_target: "{{idx.passage|string}}+{{idx.query}}_{{answers}}"
+generation_kwargs:
+  until:
+    - "</s>"
 metric_list:
   - metric: !function t5_utils.em
     aggregation: !function t5_utils.squad_em_agg

lm_eval/tasks/super_glue/rte/t5-prompt.yaml

@@ -9,6 +9,9 @@ output_type: greedy_until
 doc_to_text: "rte hypothesis: {{hypothesis}} premise: {{premise}}"
 doc_to_target: label
 doc_to_choice: ['entailment', 'not_entailment']
+generation_kwargs:
+  until:
+    - "</s>"
 metric_list:
   - metric: exact_match
     aggregation: mean

lm_eval/tasks/super_glue/wic/t5-prompt.yaml

@@ -9,6 +9,9 @@ output_type: greedy_until
 doc_to_text: "wic sentence1: {{sentence1}} sentence2: {{sentence2}} word: {{word}}"
 doc_to_target: label
 doc_to_choice: ['False', 'True']
+generation_kwargs:
+  until:
+    - "</s>"
 metric_list:
   - metric: exact_match
     aggregation: mean

lm_eval/tasks/super_glue/wsc/t5-prompt.yaml

@@ -8,6 +8,9 @@ validation_split: validation
 output_type: greedy_until
 doc_to_text: !function "t5_utils.doc_to_text"
 doc_to_target: label
+generation_kwargs:
+  until:
+    - "</s>"
 metric_list:
   - metric: accuracy
     aggregation: mean