Commit 98958aed authored by zhuwenwen's avatar zhuwenwen
Browse files

add v1 cpm support

parent c2bcb0ab
...@@ -245,8 +245,9 @@ class FastIncrementalDetokenizer(BaseIncrementalDetokenizer): ...@@ -245,8 +245,9 @@ class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
class SlowIncrementalDetokenizer(BaseIncrementalDetokenizer): class SlowIncrementalDetokenizer(BaseIncrementalDetokenizer):
def __init__(self, tokenizer: AnyTokenizer, request: EngineCoreRequest): def __init__(self, tokenizer: AnyTokenizer, request: EngineCoreRequest, mode="auto"):
super().__init__(request) super().__init__(request)
self.mode = mode
self.tokenizer = tokenizer self.tokenizer = tokenizer
params = request.sampling_params params = request.sampling_params
...@@ -283,6 +284,7 @@ class SlowIncrementalDetokenizer(BaseIncrementalDetokenizer): ...@@ -283,6 +284,7 @@ class SlowIncrementalDetokenizer(BaseIncrementalDetokenizer):
skip_special_tokens=self.skip_special_tokens, skip_special_tokens=self.skip_special_tokens,
spaces_between_special_tokens=self. spaces_between_special_tokens=self.
spaces_between_special_tokens, spaces_between_special_tokens,
mode=self.mode,
)) ))
self.tokens.extend(new_tokens) self.tokens.extend(new_tokens)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment