Commit 98958aed authored by zhuwenwen's avatar zhuwenwen
Browse files

add v1 cpm support

parent c2bcb0ab
......@@ -245,8 +245,9 @@ class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
class SlowIncrementalDetokenizer(BaseIncrementalDetokenizer):
def __init__(self, tokenizer: AnyTokenizer, request: EngineCoreRequest):
def __init__(self, tokenizer: AnyTokenizer, request: EngineCoreRequest, mode="auto"):
super().__init__(request)
self.mode = mode
self.tokenizer = tokenizer
params = request.sampling_params
......@@ -283,6 +284,7 @@ class SlowIncrementalDetokenizer(BaseIncrementalDetokenizer):
skip_special_tokens=self.skip_special_tokens,
spaces_between_special_tokens=self.
spaces_between_special_tokens,
mode=self.mode,
))
self.tokens.extend(new_tokens)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment