Fix bug in multi-token Stop Sequences (#1268)

* fix incorrect lookback protections

* bump generate_until task versions

lm_eval/tasks/gsm8k/gsm8k.yaml

@@ -34,4 +34,4 @@ filter_list:
         regex_pattern: "#### (\\-?[0-9\\.\\,]+)"
       - function: "take_first"
 metadata:
-  version: 1.0
+  version: 2.0

lm_eval/tasks/ifeval/ifeval.yaml

@@ -26,4 +26,4 @@ metric_list:
     aggregation: !function utils.agg_inst_level_acc
     higher_is_better: true
 metadata:
-  version: 1.0
+  version: 2.0

lm_eval/tasks/mgsm/direct/direct_yaml

@@ -26,4 +26,4 @@ metric_list:
     ignore_case: true
     ignore_punctuation: true
 metadata:
-  version: 0.0
+  version: 1.0

lm_eval/tasks/mgsm/en_cot/cot_yaml

@@ -28,4 +28,4 @@ filter_list:
         regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)"
       - function: "take_first"
 metadata:
-  version: 0.0
+  version: 1.0

lm_eval/tasks/mgsm/native_cot/cot_yaml

@@ -28,4 +28,4 @@ filter_list:
         regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)"
       - function: "take_first"
 metadata:
-  version: 1.0
+  version: 2.0

lm_eval/tasks/minerva_math/minerva_math_algebra.yaml

@@ -21,4 +21,4 @@ metric_list:
     higher_is_better: true
 num_fewshot: 0
 metadata:
-  version: 0.0
+  version: 1.0

lm_eval/tasks/nq_open/nq_open.yaml

@@ -29,4 +29,4 @@ metric_list:
     regexes_to_ignore:
     - "\ban|a|the\b"
 metadata:
-  version: 0.0
+  version: 1.0

lm_eval/tasks/polemo2/polemo2_in.yaml

@@ -42,4 +42,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  version: 0.0
+  version: 1.0

lm_eval/tasks/qasper/freeform.yaml

@@ -15,4 +15,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  version: 1.0
+  version: 2.0

lm_eval/tasks/scrolls/task.py

@@ -108,7 +108,7 @@ def _num_cpu_cores():
 
 
 class _SCROLLSTask(Task):
-    VERSION = 1
+    VERSION = 2
     DATASET_PATH = "tau/scrolls"
     DATASET_NAME = None
     PRUNE_TOKENIZERS = None

lm_eval/tasks/squadv2/task.py

@@ -49,7 +49,7 @@ def _squad_agg(key, items):
 
 @register_task("squadv2")
 class SQuAD2(Task):
-    VERSION = 2
+    VERSION = 3
     DATASET_PATH = "squad_v2"
     DATASET_NAME = None
 

lm_eval/tasks/translation/wmt_common_yaml

@@ -14,4 +14,4 @@ generation_kwargs:
   temperature: 0.0
 repeats: 1
 metadata:
-  version: 0.0
+  version: 1.0

lm_eval/tasks/triviaqa/default.yaml

@@ -28,4 +28,4 @@ metric_list:
     ignore_case: true
     ignore_punctuation: true
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/truthfulqa/truthfulqa_gen.yaml

@@ -76,4 +76,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/unscramble/anagrams1.yaml

@@ -17,4 +17,4 @@ metric_list:
     ignore_case: false
     ignore_punctuation: false
 metadata:
-  version: 1.0
+  version: 2.0

lm_eval/tasks/unscramble/anagrams2.yaml

@@ -17,4 +17,4 @@ metric_list:
     ignore_case: false
     ignore_punctuation: false
 metadata:
-  version: 1.0
+  version: 2.0

lm_eval/tasks/unscramble/cycle_letters.yaml

@@ -17,4 +17,4 @@ metric_list:
     ignore_case: false
     ignore_punctuation: false
 metadata:
-  version: 1.0
+  version: 2.0

lm_eval/tasks/unscramble/random_insertion.yaml

@@ -17,4 +17,4 @@ metric_list:
     ignore_case: false
     ignore_punctuation: false
 metadata:
-  version: 1.0
+  version: 2.0

lm_eval/tasks/unscramble/reversed_words.yaml

@@ -17,4 +17,4 @@ metric_list:
     ignore_case: false
     ignore_punctuation: false
 metadata:
-  version: 1.0
+  version: 2.0

lm_eval/tasks/webqs/webqs.yaml

@@ -17,4 +17,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  version: 1.0
+  version: 2.0