Skip to content

GitLab

Commit e6897273 authored by daniel-furman's avatar daniel-furman
Browse files

first stab at wrap_chat_template, remove special chars from continuation

parent 5c4d9c72
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 0 deletions
lm_eval/models/huggingface.py
View file @ e6897273
...@@ -683,6 +683,8 @@ class HFLM(LM): ...@@ -683,6 +683,8 @@ class HFLM(LM):
rfind_continuation = single_tokenized_conversation.rfind(continuation) rfind_continuation = single_tokenized_conversation.rfind(continuation)
context = single_tokenized_conversation[:rfind_continuation] context = single_tokenized_conversation[:rfind_continuation]
continuation = single_tokenized_conversation[rfind_continuation:] continuation = single_tokenized_conversation[rfind_continuation:]
# remove special chars from continuation
continuation = self.tokenizer.decode(self.tokenizer.encode(continuation), skip_special_tokens=True)
req.args = (context, continuation) req.args = (context, continuation)
new_reqs.append(req) new_reqs.append(req)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment