Skip to content

GitLab

Unverified Commit 61d3928b authored by Kamal Raj's avatar Kamal Raj Committed by GitHub
Browse files

Fix typo in FSMT Tokenizer (#20456) 

* Fix typo

* Update tokenization_fsmt.py
parent 3c39c07f
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 2 deletions
src/transformers/models/fsmt/tokenization_fsmt.py
View file @ 61d3928b
...@@ -275,8 +275,8 @@ class FSMTTokenizer(PreTrainedTokenizer): ...@@ -275,8 +275,8 @@ class FSMTTokenizer(PreTrainedTokenizer):
) )
def moses_detokenize(self, tokens, lang): def moses_detokenize(self, tokens, lang):
if lang not in self.cache_moses_tokenizer: if lang not in self.cache_moses_detokenizer:
moses_detokenizer = self.sm.MosesDetokenizer(lang=self.tgt_lang) moses_detokenizer = self.sm.MosesDetokenizer(lang=lang)
self.cache_moses_detokenizer[lang] = moses_detokenizer self.cache_moses_detokenizer[lang] = moses_detokenizer
return self.cache_moses_detokenizer[lang].detokenize(tokens) return self.cache_moses_detokenizer[lang].detokenize(tokens)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment