Unverified Commit e084089e authored by Gabriele Picco's avatar Gabriele Picco Committed by GitHub
Browse files

Fix tokenizer UnboundLocalError when padding is set to PaddingStrategy.MAX_LENGTH (#7610)

* Fix UnboundLocalError when PaddingStrategy is MAX_LENGTH

* Fix UnboundLocalError for TruncationStrategy
parent adfe6ace
...@@ -1781,6 +1781,8 @@ class PreTrainedTokenizerBase(SpecialTokensMixin): ...@@ -1781,6 +1781,8 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
padding_strategy = PaddingStrategy.LONGEST # Default to pad to the longest sequence in the batch padding_strategy = PaddingStrategy.LONGEST # Default to pad to the longest sequence in the batch
elif not isinstance(padding, PaddingStrategy): elif not isinstance(padding, PaddingStrategy):
padding_strategy = PaddingStrategy(padding) padding_strategy = PaddingStrategy(padding)
elif isinstance(padding, PaddingStrategy):
padding_strategy = padding
else: else:
padding_strategy = PaddingStrategy.DO_NOT_PAD padding_strategy = PaddingStrategy.DO_NOT_PAD
...@@ -1806,6 +1808,8 @@ class PreTrainedTokenizerBase(SpecialTokensMixin): ...@@ -1806,6 +1808,8 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
) # Default to truncate the longest sequences in pairs of inputs ) # Default to truncate the longest sequences in pairs of inputs
elif not isinstance(truncation, TruncationStrategy): elif not isinstance(truncation, TruncationStrategy):
truncation_strategy = TruncationStrategy(truncation) truncation_strategy = TruncationStrategy(truncation)
elif isinstance(truncation, TruncationStrategy):
truncation_strategy = truncation
else: else:
truncation_strategy = TruncationStrategy.DO_NOT_TRUNCATE truncation_strategy = TruncationStrategy.DO_NOT_TRUNCATE
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment