Commit 337c084b authored by daniel-furman's avatar daniel-furman
Browse files

first stab at wrap_chat_template, remove special chars tab indenting style fix

parent e6897273
......@@ -684,11 +684,13 @@ class HFLM(LM):
context = single_tokenized_conversation[:rfind_continuation]
continuation = single_tokenized_conversation[rfind_continuation:]
# remove special chars from continuation
continuation = self.tokenizer.decode(self.tokenizer.encode(continuation), skip_special_tokens=True)
continuation = self.tokenizer.decode(
self.tokenizer.encode(continuation), skip_special_tokens=True
)
req.args = (context, continuation)
new_reqs.append(req)
return new_reqs
return new_reqs
def _model_call(self, inps, attn_mask=None, labels=None):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment