Unverified Commit 996f393a authored by Funtowicz Morgan's avatar Funtowicz Morgan Committed by GitHub
Browse files

Warn the user about max_len being on the path to be deprecated. (#4528)

* Warn the user about max_len being on the path to be deprecated.

* Ensure better backward compatibility when max_len is provided to a tokenizer.

* Make sure to override the parameter and not the actual instance value.

* Format & quality
parent 0f6969b7
......@@ -22,6 +22,7 @@ import logging
import operator
import os
import re
import warnings
from collections import UserDict, defaultdict
from contextlib import contextmanager
from typing import Any, Dict, List, NamedTuple, Optional, Sequence, Tuple, Union
......@@ -822,7 +823,14 @@ class PreTrainedTokenizer(SpecialTokensMixin):
super().__init__(**kwargs)
# For backward compatibility we fallback to set model_max_length from max_len if provided
model_max_length = model_max_length if model_max_length is not None else kwargs.pop("max_len", None)
if "max_len" in kwargs:
warnings.warn(
"Parameter max_len is deprecated and will be removed in a future release. "
"Use model_max_length instead.",
category=FutureWarning,
)
model_max_length = kwargs.pop("max_len")
self.model_max_length = model_max_length if model_max_length is not None else VERY_LARGE_INTEGER
# Padding side is right by default and overridden in subclasses. If specified in the kwargs, it is changed.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment