"docs/en_US/vscode:/vscode.git/clone" did not exist on "25c4c3b5fa472a07cab61a964a1ae632b4ee5925"
Commit 337c084b authored by daniel-furman's avatar daniel-furman
Browse files

first stab at wrap_chat_template, remove special chars tab indenting style fix

parent e6897273
...@@ -684,11 +684,13 @@ class HFLM(LM): ...@@ -684,11 +684,13 @@ class HFLM(LM):
context = single_tokenized_conversation[:rfind_continuation] context = single_tokenized_conversation[:rfind_continuation]
continuation = single_tokenized_conversation[rfind_continuation:] continuation = single_tokenized_conversation[rfind_continuation:]
# remove special chars from continuation # remove special chars from continuation
continuation = self.tokenizer.decode(self.tokenizer.encode(continuation), skip_special_tokens=True) continuation = self.tokenizer.decode(
self.tokenizer.encode(continuation), skip_special_tokens=True
)
req.args = (context, continuation) req.args = (context, continuation)
new_reqs.append(req) new_reqs.append(req)
return new_reqs return new_reqs
def _model_call(self, inps, attn_mask=None, labels=None): def _model_call(self, inps, attn_mask=None, labels=None):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment