Unverified Commit 5dbf36bd authored by Yih-Dar's avatar Yih-Dar Committed by GitHub
Browse files

Fix ProphetNetTokenizer (#16082)


Co-authored-by: default avatarydshieh <ydshieh@users.noreply.github.com>
parent 923c35b5
...@@ -102,6 +102,11 @@ class ProphetNetTokenizer(PreTrainedTokenizer): ...@@ -102,6 +102,11 @@ class ProphetNetTokenizer(PreTrainedTokenizer):
pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
# first name has to correspond to main model input name
# to make sure `tokenizer.pad(...)` works correctly
# `ProphetNet` doesn't have `token_type_ids` as argument.
model_input_names: List[str] = ["input_ids", "attention_mask"]
def __init__( def __init__(
self, self,
vocab_file, vocab_file,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment