Merge remote-tracking branch 'origin/master'

.github/PULL_REQUEST_TEMPLATE.md

@@ -20,7 +20,7 @@ Fixes # (issue)
 - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/master/CONTRIBUTING.md#start-contributing-pull-requests),
       Pull Request section?
 - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link
-      to the it if that's the case.
+      to it if that's the case.
 - [ ] Did you make sure to update the documentation with your changes? Here are the
       [documentation guidelines](https://github.com/huggingface/transformers/tree/master/docs), and
       [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/master/docs#writing-source-documentation).

.github/workflows/self-push.yml

@@ -48,7 +48,7 @@ jobs:
         run: |
           source .env/bin/activate
           pip install --upgrade pip
-          pip install .[torch,sklearn,testing,onnxruntime]
+          pip install .[torch,sklearn,testing,onnxruntime,sentencepiece]
           pip install git+https://github.com/huggingface/datasets
 
       - name: Are GPUs recognized by our DL frameworks
@@ -117,7 +117,7 @@ jobs:
         run: |
           source .env/bin/activate
           pip install --upgrade pip
-          pip install .[tf,sklearn,testing,onnxruntime]
+          pip install .[tf,sklearn,testing,onnxruntime,sentencepiece]
           pip install git+https://github.com/huggingface/datasets
 
       - name: Are GPUs recognized by our DL frameworks
@@ -185,7 +185,7 @@ jobs:
         run: |
           source .env/bin/activate
           pip install --upgrade pip
-          pip install .[torch,sklearn,testing,onnxruntime]
+          pip install .[torch,sklearn,testing,onnxruntime,sentencepiece]
           pip install git+https://github.com/huggingface/datasets
 
       - name: Are GPUs recognized by our DL frameworks
@@ -244,7 +244,7 @@ jobs:
         run: |
           source .env/bin/activate
           pip install --upgrade pip
-          pip install .[tf,sklearn,testing,onnxruntime]
+          pip install .[tf,sklearn,testing,onnxruntime,sentencepiece]
           pip install git+https://github.com/huggingface/datasets
 
       - name: Are GPUs recognized by our DL frameworks

.github/workflows/self-scheduled.yml

@@ -50,7 +50,7 @@ jobs:
         run: |
           source .env/bin/activate
           pip install --upgrade pip
-          pip install .[torch,sklearn,testing,onnxruntime]
+          pip install .[torch,sklearn,testing,onnxruntime,sentencepiece]
           pip install git+https://github.com/huggingface/datasets
           pip list
 
@@ -144,7 +144,7 @@ jobs:
         run: |
           source .env/bin/activate
           pip install --upgrade pip
-          pip install .[tf,sklearn,testing,onnxruntime]
+          pip install .[tf,sklearn,testing,onnxruntime,sentencepiece]
           pip install git+https://github.com/huggingface/datasets
           pip list
 
@@ -223,7 +223,7 @@ jobs:
         run: |
           source .env/bin/activate
           pip install --upgrade pip
-          pip install .[torch,sklearn,testing,onnxruntime]
+          pip install .[torch,sklearn,testing,onnxruntime,sentencepiece]
           pip install git+https://github.com/huggingface/datasets
           pip list
 
@@ -251,11 +251,11 @@ jobs:
           RUN_SLOW: yes
         run: |
           source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s --make-reports=examples_torch_multi_gpu examples
+          python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_torch_examples_multi_gpu examples
 
       - name: Failure short reports
         if: ${{ always() }}
-        run: cat reports/examples_torch_multi_gpu_failures_short.txt
+        run: cat reports/tests_torch_examples_multi_gpu_failures_short.txt
 
       - name: Run all pipeline tests on multi-GPU
         if: ${{ always() }}
@@ -314,7 +314,7 @@ jobs:
         run: |
           source .env/bin/activate
           pip install --upgrade pip
-          pip install .[tf,sklearn,testing,onnxruntime]
+          pip install .[tf,sklearn,testing,onnxruntime,sentencepiece]
           pip install git+https://github.com/huggingface/datasets
           pip list
 
@@ -345,11 +345,11 @@ jobs:
           RUN_PIPELINE_TESTS: yes
         run: |
           source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make-reports=tests_tf_pipelines_multi_gpu tests
+          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make-reports=tests_tf_pipeline_multi_gpu tests
           
       - name: Failure short reports
         if: ${{ always() }}
-        run: cat reports/tests_tf_multi_gpu_pipelines_failures_short.txt
+        run: cat reports/tests_tf_pipeline_multi_gpu_failures_short.txt
 
       - name: Test suite reports artifacts
         if: ${{ always() }}

docs/source/model_doc/longformer.rst

@@ -99,21 +99,41 @@ Longformer specific outputs
 .. autoclass:: transformers.models.longformer.modeling_longformer.LongformerBaseModelOutputWithPooling
     :members: 
 
-.. autoclass:: transformers.models.longformer.modeling_longformer.LongformerMultipleChoiceModelOutput
+.. autoclass:: transformers.models.longformer.modeling_longformer.LongformerMaskedLMOutput
     :members: 
 
 .. autoclass:: transformers.models.longformer.modeling_longformer.LongformerQuestionAnsweringModelOutput
     :members: 
 
+.. autoclass:: transformers.models.longformer.modeling_longformer.LongformerSequenceClassifierOutput
+    :members: 
+
+.. autoclass:: transformers.models.longformer.modeling_longformer.LongformerMultipleChoiceModelOutput
+    :members: 
+
+.. autoclass:: transformers.models.longformer.modeling_longformer.LongformerTokenClassifierOutput
+    :members: 
+
 .. autoclass:: transformers.models.longformer.modeling_tf_longformer.TFLongformerBaseModelOutput
     :members: 
 
 .. autoclass:: transformers.models.longformer.modeling_tf_longformer.TFLongformerBaseModelOutputWithPooling
     :members: 
 
+.. autoclass:: transformers.models.longformer.modeling_tf_longformer.TFLongformerMaskedLMOutput
+    :members: 
+
 .. autoclass:: transformers.models.longformer.modeling_tf_longformer.TFLongformerQuestionAnsweringModelOutput
     :members: 
 
+.. autoclass:: transformers.models.longformer.modeling_tf_longformer.TFLongformerSequenceClassifierOutput
+    :members: 
+
+.. autoclass:: transformers.models.longformer.modeling_tf_longformer.TFLongformerMultipleChoiceModelOutput
+    :members: 
+
+.. autoclass:: transformers.models.longformer.modeling_tf_longformer.TFLongformerTokenClassifierOutput
+    :members: 
 
 LongformerModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -177,3 +197,24 @@ TFLongformerForQuestionAnswering
 .. autoclass:: transformers.TFLongformerForQuestionAnswering
     :members: call
 
+
+TFLongformerForSequenceClassification
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFLongformerForSequenceClassification
+    :members: call
+
+
+TFLongformerForTokenClassification
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFLongformerForTokenClassification
+    :members: call
+
+
+TFLongformerForMultipleChoice
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFLongformerForMultipleChoice
+    :members: call
+

docs/source/task_summary.rst

@@ -231,7 +231,9 @@ Here is an example of question answering using a model and a tokenizer. The proc
     ...     input_ids = inputs["input_ids"].tolist()[0]
     ...
     ...     text_tokens = tokenizer.convert_ids_to_tokens(input_ids)
-    ...     answer_start_scores, answer_end_scores = model(**inputs)
+    ...     outputs = model(**inputs)
+    ...     answer_start_scores = outputs.start_logits
+    ...     answer_end_scores = outputs.end_logits
     ...
     ...     answer_start = torch.argmax(
     ...         answer_start_scores
@@ -273,7 +275,9 @@ Here is an example of question answering using a model and a tokenizer. The proc
     ...     input_ids = inputs["input_ids"].numpy()[0]
     ...
     ...     text_tokens = tokenizer.convert_ids_to_tokens(input_ids)
-    ...     answer_start_scores, answer_end_scores = model(inputs)
+    ...     outputs = model(inputs)
+    ...     answer_start_scores = outputs.start_logits
+    ...     answer_end_scores = outputs.end_logits
     ...
     ...     answer_start = tf.argmax(
     ...         answer_start_scores, axis=1

docs/source/testing.rst

@@ -1055,7 +1055,7 @@ If you need to validate the output of a logger, you can use :obj:`CaptureLogger`
 
     msg = "Testing 1, 2, 3"
     logging.set_verbosity_info()
-    logger = logging.get_logger("transformers.tokenization_bart")
+    logger = logging.get_logger("transformers.models.bart.tokenization_bart")
     with CaptureLogger(logger) as cl:
         logger.info(msg)
     assert cl.out, msg+"\n"

examples/contrib/run_chinese_ref.py

@@ -3,7 +3,7 @@ import json
 from typing import List
 
 from ltp import LTP
-from transformers.tokenization_bert import BertTokenizer
+from transformers import BertTokenizer
 
 
 def _is_chinese_char(cp):

examples/language-modeling/README.md

@@ -90,7 +90,7 @@ selected tokens (which may be part of words), they mask randomly selected words
 to that word). This technique has been refined for Chinese in [this paper](https://arxiv.org/abs/1906.08101).
 
 To fine-tune a model using whole word masking, use the following script:
-
+```bash
 python run_mlm_wwm.py \
     --model_name_or_path roberta-base \
     --dataset_name wikitext \
@@ -164,7 +164,7 @@ context length for permutation language modeling.
 The `--max_span_length` flag may also be used to limit the length of a span of masked tokens used 
 for permutation language modeling.
 
-Here is how to fine-tun XLNet on wikitext-2:
+Here is how to fine-tune XLNet on wikitext-2:
 
 ```bash
 python run_plm.py \

examples/longform-qa/eli5_app.py

@@ -107,7 +107,12 @@ def make_support(question, source="wiki40b", method="dense", n_results=10):
     return question_doc, support_list
 
 
-@st.cache(hash_funcs={torch.Tensor: (lambda _: None), transformers.tokenization_bart.BartTokenizer: (lambda _: None)})
+@st.cache(
+    hash_funcs={
+        torch.Tensor: (lambda _: None),
+        transformers.models.bart.tokenization_bart.BartTokenizer: (lambda _: None),
+    }
+)
 def answer_question(
     question_doc, s2s_model, s2s_tokenizer, min_len=64, max_len=256, sampling=False, n_beams=2, top_p=0.95, temp=0.8
 ):

examples/rag/use_own_knowledge_dataset.py

@@ -47,7 +47,7 @@ def embed(documents: dict, ctx_encoder: DPRContextEncoder, ctx_tokenizer: DPRCon
     input_ids = ctx_tokenizer(
         documents["title"], documents["text"], truncation=True, padding="longest", return_tensors="pt"
     )["input_ids"]
-    embeddings = ctx_encoder(input_ids.to(device=device)).pooler_output
+    embeddings = ctx_encoder(input_ids.to(device=device), return_dict=True).pooler_output
     return {"embeddings": embeddings.detach().cpu().numpy()}
 
 

examples/seq2seq/distillation.py

@@ -154,7 +154,7 @@ class SummarizationDistiller(SummarizationModule):
             output_attentions=False,
             use_cache=False,
         )
-        lm_logits = student_outputs.logits
+        lm_logits = student_outputs["logits"]
 
         # Same cross entropy vs. label smoothing logic as finetune.py
         assert lm_logits.shape[-1] == self.model.config.vocab_size
@@ -171,7 +171,9 @@ class SummarizationDistiller(SummarizationModule):
         def zero_tensor():
             return torch.tensor(0.0).type_as(student_lm_loss)
 
-        teacher_enc_outputs = student_outputs.encoder_last_hidden_state  # use this unless self.different_base_models
+        teacher_enc_outputs = student_outputs[
+            "encoder_last_hidden_state"
+        ]  # use this unless self.different_base_models
         hid_loss_enc, hid_loss_dec = zero_tensor(), zero_tensor()
         if self.different_encoder:  # compute encoder hidden state loss
             all_teacher_encoder_outputs = self.teacher.get_encoder()(
@@ -180,12 +182,12 @@ class SummarizationDistiller(SummarizationModule):
                 output_hidden_states=self.do_calc_hidden_loss,
             )
             if self.different_base_models:
-                teacher_enc_outputs = all_teacher_encoder_outputs.last_hidden_state
+                teacher_enc_outputs = all_teacher_encoder_outputs["last_hidden_state"]
             elif self.do_calc_hidden_loss:
                 hid_loss_enc = self.calc_hidden_loss(
                     src_mask,
-                    student_outputs.encoder_hidden_states,
-                    all_teacher_encoder_outputs.hidden_states,
+                    student_outputs["encoder_hidden_states"],
+                    all_teacher_encoder_outputs["hidden_states"],
                     self.e_matches,
                     normalize_hidden=self.hparams.normalize_hidden,
                 )
@@ -199,12 +201,12 @@ class SummarizationDistiller(SummarizationModule):
             use_cache=False,  # since we are not passing labels, never let this default to True
         )
         dec_mask = decoder_input_ids.ne(pad_token_id)
-        loss_ce = self.calc_ce_loss(dec_mask, lm_logits, teacher_outputs.logits)
+        loss_ce = self.calc_ce_loss(dec_mask, lm_logits, teacher_outputs["logits"])
         if self.do_calc_hidden_loss:  # Intermediate supervision of decoder hidden states
             hid_loss_dec = self.calc_hidden_loss(
                 dec_mask,
-                student_outputs.decoder_hidden_states,
-                teacher_outputs.decoder_hidden_states,
+                student_outputs["decoder_hidden_states"],
+                teacher_outputs["decoder_hidden_states"],
                 self.d_matches,
                 normalize_hidden=self.hparams.normalize_hidden,
             )

examples/seq2seq/seq2seq_trainer.py

-from typing import Any, Dict, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 import torch
 from torch import nn
@@ -153,7 +153,11 @@ class Seq2SeqTrainer(Trainer):
         return loss
 
     def prediction_step(
-        self, model: nn.Module, inputs: Dict[str, Union[torch.Tensor, Any]], prediction_loss_only: bool
+        self,
+        model: nn.Module,
+        inputs: Dict[str, Union[torch.Tensor, Any]],
+        prediction_loss_only: bool,
+        ignore_keys: Optional[List[str]] = None,
     ) -> Tuple[Optional[float], Optional[torch.Tensor], Optional[torch.Tensor]]:
         """
         Perform an evaluation step on :obj:`model` using obj:`inputs`.

model_cards/Geotrend/bert-base-15lang-cased/README.md

+---
+language: multilingual
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-15lang-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+The measurements below have been computed on a [Google Cloud n1-standard-1 machine (1 vCPU, 3.75 GB)](https://cloud.google.com/compute/docs/machine-types\#n1_machine_type):
+
+|             Model               | Num parameters |   Size   |  Memory  | Loading time |
+| ------------------------------- | -------------- | -------- | -------- | ------------ |
+| bert-base-multilingual-cased    |   178 million  |  714 MB  | 1400 MB  |    4.2 sec   |
+| Geotrend/bert-base-15lang-cased |   141 million  |  564 MB  | 1098 MB  |    3.1 sec   |
+
+Handled languages: en, fr, es, de, zh, ar, ru, vi, el, bg, th, tr, hi, ur and sw.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-15lang-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-15lang-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-ar-cased/README.md

+---
+language: ar
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-ar-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-ar-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-ar-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-bg-cased/README.md

+---
+language: bg
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-bg-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-bg-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-bg-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-de-cased/README.md

+---
+language: de
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-de-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-de-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-de-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-el-cased/README.md

+---
+language: el
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-el-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-el-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-el-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-en-ar-cased/README.md

+---
+language: multilingual
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-en-ar-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-ar-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-en-ar-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-en-bg-cased/README.md

+---
+language: multilingual
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-en-bg-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-bg-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-en-bg-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-en-cased/README.md

+---
+language: en
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-en-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-en-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.