skip duplicate bos

lm_eval/models/huggingface.py

@@ -32,6 +32,7 @@ from lm_eval.api.model import TemplateLM
 from lm_eval.api.registry import register_model
 from lm_eval.models.utils import (
     Collator,
+    bos_already_added,
     clear_torch_cache,
     configure_pad_token,
     get_dtype,
@@ -901,6 +902,11 @@ class HFLM(TemplateLM):
 
         add_special_tokens = {}
         if self.backend == "causal":
+            if bos_already_added(
+                strings[0], getattr(self.tokenizer, "bos_token", None)
+            ):
+                add_special_tokens = {"add_special_tokens": False}
+            else:
                 add_special_tokens = {"add_special_tokens": False or self.add_bos_token}
 
         encoding = self.tokenizer(

lm_eval/models/utils.py

@@ -881,3 +881,7 @@ def postprocess_generated_text(
         generation = generation.split(think_end_token)[-1].lstrip()
 
     return generation
+
+
+def bos_already_added(sequence: str, bos_string: Optional[str]):
+    return sequence[0] == bos_string