first stab at wrap_chat_template, remove special chars tab indenting style fix

337c084b · daniel-furman · e6897273 · 337c084b
Commit 337c084b authored Jan 09, 2024 by daniel-furman
Show whitespace changes
Inline Side-by-side

Showing with 5 additions and 3 deletions

lm_eval/models/huggingface.py lm_eval/models/huggingface.py +5 -3

No files found.
--- a/lm_eval/models/huggingface.py
+++ b/lm_eval/models/huggingface.py
@@ -684,7 +684,9 @@ class HFLM(LM):
            context = single_tokenized_conversation[:rfind_continuation]
            continuation = single_tokenized_conversation[rfind_continuation:]
            # remove special chars from continuation
-            continuation = self.tokenizer.decode(self.tokenizer.encode(continuation), skip_special_tokens=True)
+            continuation = self.tokenizer.decode(
+                self.tokenizer.encode(continuation), skip_special_tokens=True
+            )
            req.args = (context, continuation) 
            new_reqs.append(req)