"...git@developer.sourcefind.cn:chenpangpang/transformers.git" did not exist on "fbd879219584e4a3b438d41306d68a6d85ce3da8"
Unverified Commit 5a6b138b authored by Julien Chaumond's avatar Julien Chaumond Committed by GitHub
Browse files

[Umberto] model shortcuts (#2661)

* [Umberto] model shortcuts

cc @loretoparisi @simonefrancia

see #2485

* Ensure that tokenizers will be correctly configured
parent 7fe294bf
...@@ -25,6 +25,8 @@ logger = logging.getLogger(__name__) ...@@ -25,6 +25,8 @@ logger = logging.getLogger(__name__)
CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
"camembert-base": "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-config.json", "camembert-base": "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-config.json",
"umberto-commoncrawl-cased-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/Musixmatch/umberto-commoncrawl-cased-v1/config.json",
"umberto-wikipedia-uncased-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/Musixmatch/umberto-wikipedia-uncased-v1/config.json",
} }
......
...@@ -33,6 +33,8 @@ logger = logging.getLogger(__name__) ...@@ -33,6 +33,8 @@ logger = logging.getLogger(__name__)
CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP = {
"camembert-base": "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-pytorch_model.bin", "camembert-base": "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-pytorch_model.bin",
"umberto-commoncrawl-cased-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/Musixmatch/umberto-commoncrawl-cased-v1/pytorch_model.bin",
"umberto-wikipedia-uncased-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/Musixmatch/umberto-wikipedia-uncased-v1/pytorch_model.bin",
} }
......
...@@ -40,6 +40,13 @@ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { ...@@ -40,6 +40,13 @@ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
"camembert-base": None, "camembert-base": None,
} }
SHARED_MODEL_IDENTIFIERS = [
# Load with
# `tokenizer = AutoTokenizer.from_pretrained("username/pretrained_model")`
"Musixmatch/umberto-commoncrawl-cased-v1",
"Musixmatch/umberto-wikipedia-uncased-v1",
]
class CamembertTokenizer(PreTrainedTokenizer): class CamembertTokenizer(PreTrainedTokenizer):
""" """
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment