Unverified Commit 4096000e authored by OlivierDehaene's avatar OlivierDehaene Committed by GitHub
Browse files

fix(server): fix typo in tokenizers decode (#269)

closes #268
parent 411b0d4e
...@@ -490,7 +490,7 @@ class CausalLM(Model): ...@@ -490,7 +490,7 @@ class CausalLM(Model):
def decode(self, generated_ids: List[int]) -> str: def decode(self, generated_ids: List[int]) -> str:
return self.tokenizer.decode( return self.tokenizer.decode(
generated_ids, skip_special_tokens=True, cleanup_tokenization_spaces=False generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False
) )
def forward( def forward(
......
...@@ -402,7 +402,7 @@ class FlashCausalLM(Model): ...@@ -402,7 +402,7 @@ class FlashCausalLM(Model):
def decode(self, generated_ids: Union[torch.Tensor, List[int]]) -> str: def decode(self, generated_ids: Union[torch.Tensor, List[int]]) -> str:
return self.tokenizer.decode( return self.tokenizer.decode(
generated_ids, skip_special_tokens=True, cleanup_tokenization_spaces=False generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False
) )
def forward( def forward(
......
...@@ -165,7 +165,7 @@ class FlashSantacoder(FlashCausalLM): ...@@ -165,7 +165,7 @@ class FlashSantacoder(FlashCausalLM):
def decode(self, generated_ids: List[int]) -> str: def decode(self, generated_ids: List[int]) -> str:
# Do not skip special tokens as they are used for custom parsing rules of the generated text # Do not skip special tokens as they are used for custom parsing rules of the generated text
return self.tokenizer.decode( return self.tokenizer.decode(
generated_ids, skip_special_tokens=False, cleanup_tokenization_spaces=False generated_ids, skip_special_tokens=False, clean_up_tokenization_spaces=False
) )
......
...@@ -173,7 +173,7 @@ class Galactica(OPT): ...@@ -173,7 +173,7 @@ class Galactica(OPT):
def decode(self, generated_ids: List[int]) -> str: def decode(self, generated_ids: List[int]) -> str:
# Do not skip special tokens as they are used for custom parsing rules of the generated text # Do not skip special tokens as they are used for custom parsing rules of the generated text
return self.tokenizer.decode( return self.tokenizer.decode(
generated_ids, skip_special_tokens=False, cleanup_tokenization_spaces=False generated_ids, skip_special_tokens=False, clean_up_tokenization_spaces=False
) )
def forward( def forward(
......
...@@ -64,5 +64,5 @@ class SantaCoder(CausalLM): ...@@ -64,5 +64,5 @@ class SantaCoder(CausalLM):
def decode(self, generated_ids: List[int]) -> str: def decode(self, generated_ids: List[int]) -> str:
# Do not skip special tokens as they are used for custom parsing rules of the generated text # Do not skip special tokens as they are used for custom parsing rules of the generated text
return self.tokenizer.decode( return self.tokenizer.decode(
generated_ids, skip_special_tokens=False, cleanup_tokenization_spaces=False generated_ids, skip_special_tokens=False, clean_up_tokenization_spaces=False
) )
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment