Skip to content

GitLab

Unverified Commit 0cdafbf7 authored by Stas Bekman's avatar Stas Bekman Committed by GitHub
Browse files

remove duplicated code (#7173)

parent 45b0b1ff
Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 3 deletions
src/transformers/tokenization_marian.py
View file @ 0cdafbf7
...@@ -156,9 +156,6 @@ class MarianTokenizer(PreTrainedTokenizer): ...@@ -156,9 +156,6 @@ class MarianTokenizer(PreTrainedTokenizer):
if max_target_length is not None: if max_target_length is not None:
tokenizer_kwargs["max_length"] = max_target_length tokenizer_kwargs["max_length"] = max_target_length
if max_target_length is not None:
tokenizer_kwargs["max_length"] = max_target_length
self.current_spm = self.spm_target self.current_spm = self.spm_target
model_inputs["labels"] = self(tgt_texts, **tokenizer_kwargs)["input_ids"] model_inputs["labels"] = self(tgt_texts, **tokenizer_kwargs)["input_ids"]
self.current_spm = self.spm_source self.current_spm = self.spm_source
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment