Fix bug in multi-token Stop Sequences (#1268)

* fix incorrect lookback protections

* bump generate_until task versions

lm_eval/models/huggingface.py

@@ -696,7 +696,7 @@ class HFLM(LM):
             generation_kwargs["do_sample"] = False
         # build stopping criteria
         stopping_criteria = stop_sequences_criteria(
-            self.tokenizer, stop, 1, context.shape[0]
+            self.tokenizer, stop, context.shape[1], context.shape[0]
         )
         return self.model.generate(
             input_ids=context,

lm_eval/tasks/babi/babi.yaml

@@ -17,4 +17,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  version: 0.0
+  version: 1.0

lm_eval/tasks/bbh/cot_fewshot/_cot_fewshot_template_yaml

@@ -27,4 +27,4 @@ filter_list:
       - function: "take_first"
 num_fewshot: 0
 metadata:
-  version: 1.0
+  version: 2.0

lm_eval/tasks/bbh/cot_zeroshot/_cot_zeroshot_template_yaml

@@ -24,4 +24,4 @@ filter_list:
       - function: "take_first"
 num_fewshot: 0
 metadata:
-  version: 0
+  version: 1.0

lm_eval/tasks/bbh/fewshot/_fewshot_template_yaml

@@ -18,4 +18,4 @@ generation_kwargs:
   temperature: 0.0
 num_fewshot: 0
 metadata:
-  version: 0
+  version: 1.0

lm_eval/tasks/bbh/zeroshot/_zeroshot_template_yaml

@@ -18,4 +18,4 @@ generation_kwargs:
   temperature: 0.0
 num_fewshot: 0
 metadata:
-  version: 0
+  version: 1.0

lm_eval/tasks/benchmarks/flan/yaml_templates/cot_template_yaml

@@ -17,3 +17,5 @@ filter_list:
       - function: "regex"
         regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)"
       - function: "take_first"
+metadata:
+  version: 1.0

lm_eval/tasks/benchmarks/flan/yaml_templates/held_in_template_yaml

@@ -9,3 +9,5 @@ generation_kwargs:
     - "</s>"
   do_sample: false
   temperature: 0.0
+metadata:
+  version: 1.0

lm_eval/tasks/bigbench/generate_until_template_yaml

@@ -15,4 +15,4 @@ metric_list:
     higher_is_better: true
     ignore_punctuation: true
 metadata:
-  version: 0.0
+  version: 1.0

lm_eval/tasks/code_x_glue/code-text/go.yaml

@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 0.0
+  version: 1.0

lm_eval/tasks/code_x_glue/code-text/java.yaml

@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 0.0
+  version: 1.0

lm_eval/tasks/code_x_glue/code-text/javascript.yaml

@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 0.0
+  version: 1.0

lm_eval/tasks/code_x_glue/code-text/php.yaml

@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 0.0
+  version: 1.0

lm_eval/tasks/code_x_glue/code-text/python.yaml

@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 0.0
+  version: 1.0

lm_eval/tasks/code_x_glue/code-text/ruby.yaml

@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/coqa/default.yaml

@@ -19,4 +19,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/drop/default.yaml

@@ -21,4 +21,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/fld/fld_default.yaml

@@ -18,4 +18,4 @@ filter_list:
       - function: remove_whitespace
       - function: take_first
 metadata:
-  version: 1.0
+  version: 2.0

lm_eval/tasks/gsm8k/gsm8k-cot-self-consistency.yaml

@@ -31,4 +31,4 @@ filter_list:
       - function: "majority_vote"
       - function: "take_first"
 metadata:
-  version: 0.0
+  version: 1.0

lm_eval/tasks/gsm8k/gsm8k-cot.yaml

@@ -41,4 +41,4 @@ filter_list:
         regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)."
       - function: "take_first"
 metadata:
-  version: 0.0
+  version: 1.0