Refactor Code samples; Test code samples (#5036)

* Refactor code samples

* Test docstrings

* Style

* Tokenization examples

* Run rust of tests

* First step to testing source docs

* Style and BART comment

* Test the remainder of the code samples

* Style

* let to const

* Formatting fixes

* Ready for merge

* Fix fixture + Style

* Fix last tests

* Update docs/source/quicktour.rst
Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>

* Addressing @sgugger's comments + Fix MobileBERT in TF
Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>

docs/source/_static/css/code-snippets.css

@@ -10,3 +10,7 @@
 .highlight .kn, .highlight .nv, .highlight .s2, .highlight .ow {
     color: #6670FF;
 }
+
+.highlight .gp {
+    color: #FB8D68;
+}
\ No newline at end of file

docs/source/_static/css/huggingface.css

@@ -44,6 +44,7 @@
     display: flex;
     flex-direction: row;
     justify-content: flex-end;
+    margin-right: 30px;
 }
 
 .framework-selector > button {
@@ -60,6 +61,12 @@
     padding: 5px;
 }
 
+/* Copy button */
+
+a.copybtn {
+    margin: 3px;
+}
+
 /* The literal code blocks */
 .rst-content tt.literal, .rst-content tt.literal, .rst-content code.literal {
     color: #6670FF;

docs/source/_static/js/custom.js

@@ -157,6 +157,8 @@ function platformToggle() {
     const codeBlocks = Array.from(document.getElementsByClassName("highlight"));
     const pytorchIdentifier = "## PYTORCH CODE";
     const tensorflowIdentifier = "## TENSORFLOW CODE";
+
+    const promptSpanIdentifier = `<span class="gp">&gt;&gt;&gt; </span>`
     const pytorchSpanIdentifier = `<span class="c1">${pytorchIdentifier}</span>`;
     const tensorflowSpanIdentifier = `<span class="c1">${tensorflowIdentifier}</span>`;
 
@@ -169,10 +171,22 @@ function platformToggle() {
         let tensorflowSpans;
 
         if(pytorchSpanPosition < tensorflowSpanPosition){
-            pytorchSpans = spans.slice(pytorchSpanPosition + pytorchSpanIdentifier.length + 1, tensorflowSpanPosition);
+            const isPrompt = spans.slice(
+                spans.indexOf(tensorflowSpanIdentifier) - promptSpanIdentifier.length,
+                spans.indexOf(tensorflowSpanIdentifier)
+            ) == promptSpanIdentifier;
+            const finalTensorflowSpanPosition = isPrompt ? tensorflowSpanPosition - promptSpanIdentifier.length : tensorflowSpanPosition;
+
+            pytorchSpans = spans.slice(pytorchSpanPosition + pytorchSpanIdentifier.length + 1, finalTensorflowSpanPosition);
             tensorflowSpans = spans.slice(tensorflowSpanPosition + tensorflowSpanIdentifier.length + 1, spans.length);
         }else{
-            tensorflowSpans = spans.slice(tensorflowSpanPosition + tensorflowSpanIdentifier.length + 1, pytorchSpanPosition);
+            const isPrompt = spans.slice(
+                spans.indexOf(pytorchSpanIdentifier) - promptSpanIdentifier.length,
+                spans.indexOf(pytorchSpanIdentifier)
+            ) == promptSpanIdentifier;
+            const finalPytorchSpanPosition = isPrompt ? pytorchSpanPosition - promptSpanIdentifier.length : pytorchSpanPosition;
+
+            tensorflowSpans = spans.slice(tensorflowSpanPosition + tensorflowSpanIdentifier.length + 1, finalPytorchSpanPosition);
             pytorchSpans = spans.slice(pytorchSpanPosition + pytorchSpanIdentifier.length + 1, spans.length);
         }
 

docs/source/conf.py

@@ -44,7 +44,8 @@ extensions = [
     'sphinx.ext.napoleon',
     'recommonmark',
     'sphinx.ext.viewcode',
-    'sphinx_markdown_tables'
+    'sphinx_markdown_tables',
+    'sphinx_copybutton'
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -74,6 +75,8 @@ exclude_patterns = [u'_build', 'Thumbs.db', '.DS_Store']
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = None
 
+# Remove the prompt when copying examples
+copybutton_prompt_text = ">>> "
 
 # -- Options for HTML output -------------------------------------------------
 

docs/source/glossary.rst

@@ -45,17 +45,16 @@ tokenizer, which is a `WordPiece <https://arxiv.org/pdf/1609.08144.pdf>`__ token
 
 ::
 
-    from transformers import BertTokenizer
-    tokenizer = BertTokenizer.from_pretrained("bert-base-cased")
+    >>> from transformers import BertTokenizer
+    >>> tokenizer = BertTokenizer.from_pretrained("bert-base-cased")
 
-    sequence = "A Titan RTX has 24GB of VRAM"
+    >>> sequence = "A Titan RTX has 24GB of VRAM"
 
 The tokenizer takes care of splitting the sequence into tokens available in the tokenizer vocabulary.
 
 ::
 
-    tokenized_sequence = tokenizer.tokenize(sequence)
-    print(tokenized_sequence)
+    >>> tokenized_sequence = tokenizer.tokenize(sequence)
 
 The tokens are either words or subwords. Here for instance, "VRAM" wasn't in the model vocabulary, so it's been split
 in "V", "RA" and "M". To indicate those tokens are not separate words but parts of the same word, a double-dash is
@@ -63,6 +62,7 @@ added for "RA" and "M":
 
 ::
 
+    >>> print(tokenized_sequence)
     ['A', 'Titan', 'R', '##T', '##X', 'has', '24', '##GB', 'of', 'V', '##RA', '##M']
 
 These tokens can then be converted into IDs which are understandable by the model. This can be done by directly feeding
@@ -71,14 +71,14 @@ the sentence to the tokenizer, which leverages the Rust implementation of
 
 ::
 
-    encoded_sequence = tokenizer(sequence)["input_ids"]
-    print(encoded_sequence)
+    >>> encoded_sequence = tokenizer(sequence)["input_ids"]
 
 The tokenizer returns a dictionary with all the arguments necessary for its corresponding model to work properly. The
 token indices are under the key "input_ids":
 
 ::
 
+    >>> print(encoded_sequence)
     [101, 138, 18696, 155, 1942, 3190, 1144, 1572, 13745, 1104, 159, 9664, 2107, 102]
 
 Note that the tokenizer automatically adds "special tokens" (if the associated model rely on them) which are special
@@ -86,13 +86,14 @@ IDs the model sometimes uses. If we decode the previous sequence of ids,
 
 ::
 
-    tokenizer.decode(encoded_sequence)
+    >>> decoded_sequence = tokenizer.decode(encoded_sequence)
 
 we will see 
 
 ::
 
-    '[CLS] A Titan RTX has 24GB of VRAM [SEP]'
+    >>> print(decoded_sequence)
+    [CLS] A Titan RTX has 24GB of VRAM [SEP]
 
 because this is the way a :class:`~transformers.BertModel` is going to expect its inputs.
 
@@ -108,21 +109,20 @@ For example, consider these two sequences:
 
 ::
 
-    from transformers import BertTokenizer
-    tokenizer = BertTokenizer.from_pretrained("bert-base-cased")
+    >>> from transformers import BertTokenizer
+    >>> tokenizer = BertTokenizer.from_pretrained("bert-base-cased")
 
-    sequence_a = "This is a short sequence."
-    sequence_b = "This is a rather long sequence. It is at least longer than the sequence A."
+    >>> sequence_a = "This is a short sequence."
+    >>> sequence_b = "This is a rather long sequence. It is at least longer than the sequence A."
 
-    encoded_sequence_a = tokenizer(sequence_a)["input_ids"]
-    encoded_sequence_b = tokenizer(sequence_b)["input_ids"]
-    
-    len(encoded_sequence_a), len(encoded_sequence_b)
+    >>> encoded_sequence_a = tokenizer(sequence_a)["input_ids"]
+    >>> encoded_sequence_b = tokenizer(sequence_b)["input_ids"]
 
 The encoded versions have different lengths:
 
 ::
 
+    >>> len(encoded_sequence_a), len(encoded_sequence_b)
     (8, 19)
 
 Therefore, we can't be put then together in a same tensor as-is. The first sequence needs to be padded up to the length
@@ -133,15 +133,14 @@ it to pad like this:
 
 ::
 
-    padded_sequences = tokenizer([sequence_a, sequence_b], padding=True)
-    padded_sequences["input_ids"]
+    >>> padded_sequences = tokenizer([sequence_a, sequence_b], padding=True)
 
 We can see that 0s have been added on the right of the first sentence to make it the same length as the second one:
 
 ::
 
-    [[101, 1188, 1110, 170, 1603, 4954, 119, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-     [101, 1188, 1110, 170, 1897, 1263, 4954, 119, 1135, 1110, 1120, 1655, 2039, 1190, 1103, 4954, 138, 119, 102]]
+    >>> padded_sequences["input_ids"]
+    [[101, 1188, 1110, 170, 1603, 4954, 119, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [101, 1188, 1110, 170, 1897, 1263, 4954, 119, 1135, 1110, 1120, 1655, 2039, 1190, 1103, 4954, 138, 119, 102]]
 
 This can then be converted into a tensor in PyTorch or TensorFlow. The attention mask is a binary tensor indicating
 the position of the padded indices so that the model does not attend to them. For the
@@ -150,14 +149,8 @@ a padded value. This attention mask is in the dictionary returned by the tokeniz
 
 ::
 
-    padded_sequences["attention_mask"]
-
-will give back
-
-::
-
-    [[1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-     [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]
+    >>> padded_sequences["attention_mask"]
+    [[1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]
 
 .. _token-type-ids:
 
@@ -170,26 +163,27 @@ tokens. For example, the BERT model builds its two sequence input as such:
 
 ::
 
-   # [CLS] SEQUENCE_A [SEP] SEQUENCE_B [SEP]
+   >>> # [CLS] SEQUENCE_A [SEP] SEQUENCE_B [SEP]
 
 We can use our tokenizer to automatically generate such a sentence by passing the two sequences as two arguments (and
 not a list like before) like this:
 
 ::
 
-    from transformers import BertTokenizer
-    tokenizer = BertTokenizer.from_pretrained("bert-base-cased")
-    sequence_a = "HuggingFace is based in NYC"
-    sequence_b = "Where is HuggingFace based?"
+    >>> from transformers import BertTokenizer
+    >>> tokenizer = BertTokenizer.from_pretrained("bert-base-cased")
+    >>> sequence_a = "HuggingFace is based in NYC"
+    >>> sequence_b = "Where is HuggingFace based?"
 
-    encoded_dict = tokenizer(sequence_a, sequence_b)
-    tokenizer.decode(encoded_dict["input_ids"])
+    >>> encoded_dict = tokenizer(sequence_a, sequence_b)
+    >>> decoded = tokenizer.decode(encoded_dict["input_ids"])
 
 which will return:
 
 ::
 
-    "[CLS] HuggingFace is based in NYC [SEP] Where is HuggingFace based? [SEP]"
+    >>> print(decoded)
+    [CLS] HuggingFace is based in NYC [SEP] Where is HuggingFace based? [SEP]
 
 This is enough for some models to understand where one sequence ends and where another begins. However, other models
 such as BERT have an additional mechanism, which are the token type IDs (also called segment IDs). They are a binary
@@ -199,12 +193,7 @@ The tokenizer returns in the dictionary under the key "token_type_ids":
 
 ::
 
-    encoded_dict['token_type_ids']
-
-will return
-
-::
-
+    >>> encoded_dict['token_type_ids']
     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1]
 
 The first sequence, the "context" used for the question, has all its tokens represented by :obj:`0`, whereas the

docs/source/multilingual.rst

@@ -36,10 +36,11 @@ Here is an example using the ``xlm-clm-enfr-1024`` checkpoint (Causal language m
 
 .. code-block::
 
-    import torch
-    from transformers import XLMTokenizer, XLMWithLMHeadModel
+    >>> import torch
+    >>> from transformers import XLMTokenizer, XLMWithLMHeadModel
 
-    tokenizer = XLMTokenizer.from_pretrained("xlm-clm-1024-enfr")
+    >>> tokenizer = XLMTokenizer.from_pretrained("xlm-clm-enfr-1024")
+    >>> model = XLMWithLMHeadModel.from_pretrained("xlm-clm-enfr-1024")
 
 
 The different languages this model/tokenizer handles, as well as the ids of these languages are visible using the
@@ -47,16 +48,15 @@ The different languages this model/tokenizer handles, as well as the ids of thes
 
 .. code-block::
 
-    # Continuation of the previous script
-    print(tokenizer.lang2id)  # {'en': 0, 'fr': 1}
+    >>> print(tokenizer.lang2id)
+    {'en': 0, 'fr': 1}
 
 
 These ids should be used when passing a language parameter during a model pass. Let's define our inputs:
 
 .. code-block::
 
-    # Continuation of the previous script
-    input_ids = torch.tensor([tokenizer.encode("Wikipedia was used to")]) # batch size of 1
+    >>> input_ids = torch.tensor([tokenizer.encode("Wikipedia was used to")]) # batch size of 1
 
 
 We should now define the language embedding by using the previously defined language id. We want to create a tensor
@@ -64,20 +64,18 @@ filled with the appropriate language ids, of the same size as input_ids. For eng
 
 .. code-block::
 
-    # Continuation of the previous script
-    language_id = tokenizer.lang2id['en']  # 0
-    langs = torch.tensor([language_id] * input_ids.shape[1])  # torch.tensor([0, 0, 0, ..., 0])
+    >>> language_id = tokenizer.lang2id['en']  # 0
+    >>> langs = torch.tensor([language_id] * input_ids.shape[1])  # torch.tensor([0, 0, 0, ..., 0])
 
-    # We reshape it to be of size (batch_size, sequence_length)
-    langs = langs.view(1, -1) # is now of shape [1, sequence_length] (we have a batch size of 1)
+    >>> # We reshape it to be of size (batch_size, sequence_length)
+    >>> langs = langs.view(1, -1) # is now of shape [1, sequence_length] (we have a batch size of 1)
 
 
 You can then feed it all as input to your model:
 
 .. code-block::
 
-    # Continuation of the previous script
-    outputs = model(input_ids, langs=langs)
+    >>> outputs = model(input_ids, langs=langs)
 
 
 The example `run_generation.py <https://github.com/huggingface/transformers/blob/master/examples/text-generation/run_generation.py>`__

docs/source/quicktour.rst

@@ -32,40 +32,33 @@ provides the following tasks out of the box:
 Let's see how this work for sentiment analysis (the other tasks are all covered in the
 :doc:`task summary </task_summary>`):
 
-::
+.. code-block::
 
-    from transformers import pipeline
-    classifier = pipeline('sentiment-analysis')
+    >>> from transformers import pipeline
+    >>> classifier = pipeline('sentiment-analysis')
 
 When typing this command for the first time, a pretrained model and its tokenizer are downloaded and cached. We will
 look at both later on, but as an introduction the tokenizer's job is to preprocess the text for the model, which is
 then responsible for making predictions. The pipeline groups all of that together, and post-process the predictions to
-make them readable. For instance
-
-::
+make them readable. For instance:
 
-    classifier('We are very happy to show you the 🤗 Transformers library.')
 
-will return something like this:
-
-::
+.. code-block::
 
+    >>> classifier('We are very happy to show you the 🤗 Transformers library.')
     [{'label': 'POSITIVE', 'score': 0.9997795224189758}]
 
 That's encouraging! You can use it on a list of sentences, which will be preprocessed then fed to the model as a
-`batch`:
-
-::
+`batch`, returning a list of dictionaries like this one:
 
-    classifier(["We are very happy to show you the 🤗 Transformers library.",
-                "We hope you don't hate it."])
+.. code-block::
 
-returning a list of dictionaries like this one:
-
-::
-
-    [{'label': 'POSITIVE', 'score': 0.9997795224189758},
-     {'label': 'NEGATIVE', 'score': 0.5308589935302734}]
+    >>> results = classifier(["We are very happy to show you the 🤗 Transformers library.",
+    ...            "We hope you don't hate it."])
+    >>> for result in results:
+    ...     print(f"label: {result['label']}, with score: {round(result['score'], 4)}")
+    label: POSITIVE, with score: 0.9998
+    label: NEGATIVE, with score: 0.5309
 
 You can see the second sentence has been classified as negative (it needs to be positive or negative) but its score is
 fairly neutral.
@@ -83,9 +76,9 @@ see how we can use it.
 
 You can directly pass the name of the model to use to :func:`~transformers.pipeline`:
 
-::
+.. code-block::
 
-    classifier = pipeline('sentiment-analysis', model="nlptown/bert-base-multilingual-uncased-sentiment")
+    >>> classifier = pipeline('sentiment-analysis', model="nlptown/bert-base-multilingual-uncased-sentiment")
 
 This classifier can now deal with texts in English, French, but also Dutch, German, Italian and Spanish! You can also
 replace that name by a local folder where you have saved a pretrained model (see below). You can also pass a model
@@ -98,29 +91,30 @@ tokenizer associated to the model we picked and instantiate it. The second is
 the model itself. Note that if we were using the library on an other task, the class of the model would change. The
 :doc:`task summary </task_summary>` tutorial summarizes which class is used for which task.
 
-::
+.. code-block::
 
-    ## PYTORCH CODE
-    from transformers import AutoTokenizer, AutoModelForSequenceClassification
-    ## TENSORFLOW CODE
-    from transformers import AutoTokenizer, TFAutoModelForSequenceClassification
+    >>> ## PYTORCH CODE
+    >>> from transformers import AutoTokenizer, AutoModelForSequenceClassification
+    >>> ## TENSORFLOW CODE
+    >>> from transformers import AutoTokenizer, TFAutoModelForSequenceClassification
 
 Now, to download the models and tokenizer we found previously, we just have to use the
 :func:`~transformers.AutoModelForSequenceClassification.from_pretrained` method (feel free to replace ``model_name`` by
 any other model from the model hub):
 
-::
+.. code-block::
 
-    ## PYTORCH CODE
-    model_name = "nlptown/bert-base-multilingual-uncased-sentiment"
-    model = AutoModelForSequenceClassification.from_pretrained(model_name)
-    tokenizer = AutoTokenizer.from_pretrained(model_name)
-    pipe = pipeline('sentiment-analysis', model=model, tokenizer=tokenizer)
-    ## TENSORFLOW CODE
-    model_name = "nlptown/bert-base-multilingual-uncased-sentiment"
-    model = TFAutoModelForSequenceClassification.from_pretrained(model_name)
-    tokenizer = AutoTokenizer.from_pretrained(model_name)
-    classifier = pipeline('sentiment-analysis', model=model, tokenizer=tokenizer)
+    >>> ## PYTORCH CODE
+    >>> model_name = "nlptown/bert-base-multilingual-uncased-sentiment"
+    >>> model = AutoModelForSequenceClassification.from_pretrained(model_name)
+    >>> tokenizer = AutoTokenizer.from_pretrained(model_name)
+    >>> pipe = pipeline('sentiment-analysis', model=model, tokenizer=tokenizer)
+    >>> ## TENSORFLOW CODE
+    >>> model_name = "nlptown/bert-base-multilingual-uncased-sentiment"
+    >>> # This model only exists in PyTorch, so we use the `from_pt` flag to import that model in TensorFlow.
+    >>> model = TFAutoModelForSequenceClassification.from_pretrained(model_name, from_pt=True) 
+    >>> tokenizer = AutoTokenizer.from_pretrained(model_name)
+    >>> classifier = pipeline('sentiment-analysis', model=model, tokenizer=tokenizer)
 
 If you don't find a model that has been pretrained on some data similar to yours, you will need to fine-tune a
 pretrained model on your data. We provide :doc:`example scripts </examples>` to do so. Once you're done, don't forget
@@ -136,16 +130,16 @@ using the :obj:`from_pretrained` method:
 
 ::
 
-    ## PYTORCH CODE
-    from transformers import AutoTokenizer, AutoModelForSequenceClassification
-    model_name = "distilbert-base-uncased-finetuned-sst-2-english"
-    model = AutoModelForSequenceClassification.from_pretrained(model_name)
-    tokenizer = AutoTokenizer.from_pretrained(model_name)
-    ## TENSORFLOW CODE
-    from transformers import AutoTokenizer, TFAutoModelForSequenceClassification
-    model_name = "distilbert-base-uncased-finetuned-sst-2-english"
-    model = TFAutoModelForSequenceClassification.from_pretrained(model_name)
-    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    >>> ## PYTORCH CODE
+    >>> from transformers import AutoTokenizer, AutoModelForSequenceClassification
+    >>> model_name = "distilbert-base-uncased-finetuned-sst-2-english"
+    >>> pt_model = AutoModelForSequenceClassification.from_pretrained(model_name)
+    >>> tokenizer = AutoTokenizer.from_pretrained(model_name)
+    >>> ## TENSORFLOW CODE
+    >>> from transformers import AutoTokenizer, TFAutoModelForSequenceClassification
+    >>> model_name = "distilbert-base-uncased-finetuned-sst-2-english"
+    >>> tf_model = TFAutoModelForSequenceClassification.from_pretrained(model_name)
+    >>> tokenizer = AutoTokenizer.from_pretrained(model_name)
 
 Using the tokenizer
 ^^^^^^^^^^^^^^^^^^^
@@ -161,48 +155,56 @@ the model. To do this, the tokenizer has a `vocab`, which is the part we downloa
 
 To apply these steps on a given text, we can just feed it to our tokenizer:
 
-::
+.. code-block::
 
-    input = tokenizer("We are very happy to show you the 🤗 Transformers library.")
-    print(input)
+    >>> inputs = tokenizer("We are very happy to show you the 🤗 Transformers library.")
 
 This returns a dictionary string to list of ints. It contains the `ids of the tokens <glossary.html#input-ids>`__,
 as mentioned before, but also additional arguments that will be useful to the model. Here for instance, we also have an
 `attention mask <glossary.html#attention-mask>`__ that the model will use to have a better understanding of the sequence:
 
 
-::
-    {'input_ids': [101, 2057, 2024, 2200, 3407, 2000, 2265, 2017, 1996, 100, 19081, 3075, 1012, 102],
-     'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}
+.. code-block::
+
+    >>> print(inputs)
+    {'input_ids': [101, 2057, 2024, 2200, 3407, 2000, 2265, 2017, 1996, 100, 19081, 3075, 1012, 102], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}
 
 You can pass a list of sentences directly to your tokenizer. If your goal is to send them through your model as a
 batch, you probably want to pad them all to the same length, truncate them to the maximum length the model can accept
 and get tensors back. You can specify all of that to the tokenizer:
 
-::
-
-    ## PYTORCH CODE
-    batch = tokenizer(
-        ["We are very happy to show you the 🤗 Transformers library.",
-         "We hope you don't hate it."],
-        padding=True, truncation=True, return_tensors="pt")
-    print(batch)
-    ## TENSORFLOW CODE
-    batch = tokenizer(
-        ["We are very happy to show you the 🤗 Transformers library.",
-         "We hope you don't hate it."],
-        padding=True, truncation=True, return_tensors="tf")
-    print(batch)
+.. code-block::
+
+    >>> ## PYTORCH CODE
+    >>> pt_batch = tokenizer(
+    ...     ["We are very happy to show you the 🤗 Transformers library.", "We hope you don't hate it."],
+    ...     padding=True,
+    ...     truncation=True,
+    ...     return_tensors="pt"
+    ... )
+    >>> ## TENSORFLOW CODE
+    >>> tf_batch = tokenizer(
+    ...     ["We are very happy to show you the 🤗 Transformers library.", "We hope you don't hate it."],
+    ...     padding=True,
+    ...     truncation=True,
+    ...     return_tensors="tf"
+    ... )
 
 The padding is automatically applied on the side the model expect it (in this case, on the right), with the
 padding token the model was pretrained with. The attention mask is also adapted to take the padding into account:
 
-::
+.. code-block::
 
-    {'input_ids': tensor([[  101,  2057,  2024,  2200,  3407,  2000,  2265,  2017,  1996,   100, 19081,  3075,  1012,   102],
-                          [  101,  2057,  3246,  2017,  2123,  1005,  1056,  5223,  2009,  1012,   102,     0,     0,     0]]), 
-     'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
-                               [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0]])}
+    >>> ## PYTORCH CODE
+    >>> for key, value in pt_batch.items():
+    ...     print(f"{key}: {value.numpy().tolist()}")
+    input_ids: [[101, 2057, 2024, 2200, 3407, 2000, 2265, 2017, 1996, 100, 19081, 3075, 1012, 102], [101, 2057, 3246, 2017, 2123, 1005, 1056, 5223, 2009, 1012, 102, 0, 0, 0]]
+    attention_mask: [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0]]
+    >>> ## TENSORFLOW CODE
+    >>> for key, value in tf_batch.items():
+    ...     print(f"{key}: {value.numpy().tolist()}")
+    input_ids: [[101, 2057, 2024, 2200, 3407, 2000, 2265, 2017, 1996, 100, 19081, 3075, 1012, 102], [101, 2057, 3246, 2017, 2123, 1005, 1056, 5223, 2009, 1012, 102, 0, 0, 0]]
+    attention_mask: [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0]]
 
 You can learn more about tokenizers :doc:`here <preprocessing>`.
 
@@ -213,20 +215,27 @@ Once your input has been preprocessed by the tokenizer, you can directly send it
 contain all the relevant information the model needs. If you're using a TensorFlow model, you can directly pass the
 dictionary keys to tensor, for a PyTorch model, you need to unpack the dictionary by adding :obj:`**`.
 
-::
+.. code-block::
 
-    ## PYTORCH CODE
-    outputs = model(**batch)
-    ## TENSORFLOW CODE
-    outputs = model(batch)
+    >>> ## PYTORCH CODE
+    >>> pt_outputs = pt_model(**pt_batch)
+    >>> ## TENSORFLOW CODE
+    >>> tf_outputs = tf_model(tf_batch)
 
 In 🤗 Transformers, all outputs are tuples (with only one element potentially). Here, we get a tuple with just the
 final activations of the model.
 
-::
+.. code-block::
 
-    (tensor([[-4.1329,  4.3811],
-             [ 0.0818, -0.0418]]),)
+    >>> ## PYTORCH CODE
+    >>> print(pt_outputs)
+    (tensor([[-4.0833,  4.3364],
+            [ 0.0818, -0.0418]], grad_fn=<AddmmBackward>),)
+    >>> ## TENSORFLOW CODE
+    >>> print(tf_outputs)
+    (<tf.Tensor: shape=(2, 2), dtype=float32, numpy=
+    array([[-4.0832963 ,  4.3364134 ],
+           [ 0.08181238, -0.04178794]], dtype=float32)>,)
 
 .. note::
 
@@ -235,33 +244,39 @@ final activations of the model.
 
 Let's apply the SoftMax activation to get predictions.
 
-::
+.. code-block::
 
-    ## PYTORCH CODE
-    import torch.nn.functional as F
-    predictions = F.softmax(outputs[0], dim=-1)
-    print(predictions)
-    ## TENSORFLOW CODE
-    predictions = tf.nn.softmax(outputs[0], axis=-1)
-    print(predictions)
+    >>> ## PYTORCH CODE
+    >>> import torch.nn.functional as F
+    >>> pt_predictions = F.softmax(pt_outputs[0], dim=-1)
+    >>> ## TENSORFLOW CODE
+    >>> import tensorflow as tf
+    >>> tf_predictions = tf.nn.softmax(tf_outputs[0], axis=-1)
 
 We can see we get the numbers from before:
 
-::
+.. code-block::
 
-    tensor([[2.0060e-04, 9.9980e-01],
-            [5.3086e-01, 4.6914e-01]])
+    >>> ## TENSORFLOW CODE
+    >>> print(tf_predictions)
+    tf.Tensor(
+    [[2.2042994e-04 9.9977952e-01]
+     [5.3086078e-01 4.6913919e-01]], shape=(2, 2), dtype=float32)
+    >>> ## PYTORCH CODE
+    >>> print(pt_predictions)
+    tensor([[2.2043e-04, 9.9978e-01],
+            [5.3086e-01, 4.6914e-01]], grad_fn=<SoftmaxBackward>)
 
 If you have labels, you can provide them to the model, it will return a tuple with the loss and the final activations.
 
-::
+.. code-block::
 
-    ## PYTORCH CODE
-    import torch
-    outputs = model(**batch, labels = torch.tensor([1, 0])
-    ## TENSORFLOW CODE
-    import tensorflow as tf
-    outputs = model(batch, labels = tf.constant([1, 0])
+    >>> ## PYTORCH CODE
+    >>> import torch
+    >>> pt_outputs = pt_model(**pt_batch, labels = torch.tensor([1, 0]))
+    >>> ## TENSORFLOW CODE
+    >>> import tensorflow as tf
+    >>> tf_outputs = tf_model(tf_batch, labels = tf.constant([1, 0]))
 
 Models are standard `torch.nn.Module <https://pytorch.org/docs/stable/nn.html#torch.nn.Module>`__ or
 `tf.keras.Model <https://www.tensorflow.org/api_docs/python/tf/keras/Model>`__ so you can use them in your usual
@@ -298,12 +313,12 @@ Lastly, you can also ask the model to return all hidden states and all attention
 
 ::
 
-    ## PYTORCH CODE
-    outputs = model(**batch, output_hidden_states=True, output_attentions=True)
-    all_hidden_states, all_attentions = outputs[-2:]
-    ## TENSORFLOW CODE
-    outputs = model(batch, output_hidden_states=True, output_attentions=True)
-    all_hidden_states, all_attentions = outputs[-2:]
+    >>> ## PYTORCH CODE
+    >>> pt_outputs = pt_model(**pt_batch, output_hidden_states=True, output_attentions=True)
+    >>> all_hidden_states, all_attentions = pt_outputs[-2:]
+    >>> ## TENSORFLOW CODE
+    >>> tf_outputs = tf_model(tf_batch, output_hidden_states=True, output_attentions=True)
+    >>> all_hidden_states, all_attentions = tf_outputs[-2:]
 
 Accessing the code
 ^^^^^^^^^^^^^^^^^^
@@ -318,18 +333,18 @@ using the :doc:`DistilBERT </model_doc/distilbert>` architecture. The model auto
 to that specific model, or browse the source code. This is how you would directly instantiate model and tokenizer
 without the auto magic:
 
-::
+.. code-block::
 
-    ## PYTORCH CODE
-    from transformers import DistilBertTokenizer, DistilBertForSequenceClassification
-    model_name = "distilbert-base-uncased-finetuned-sst-2-english"
-    model = DistilBertForSequenceClassification.from_pretrained(model_name)
-    tokenizer = DistilBertTokenizer.from_pretrained(model_name)
-    ## TENSORFLOW CODE
-    from transformers import DistilBertTokenizer, TFDistilBertForSequenceClassification
-    model_name = "distilbert-base-uncased-finetuned-sst-2-english"
-    model = TFDistilBertForSequenceClassification.from_pretrained(model_name)
-    tokenizer = DistilBertTokenizer.from_pretrained(model_name)
+    >>> ## PYTORCH CODE
+    >>> from transformers import DistilBertTokenizer, DistilBertForSequenceClassification
+    >>> model_name = "distilbert-base-uncased-finetuned-sst-2-english"
+    >>> model = DistilBertForSequenceClassification.from_pretrained(model_name)
+    >>> tokenizer = DistilBertTokenizer.from_pretrained(model_name)
+    >>> ## TENSORFLOW CODE
+    >>> from transformers import DistilBertTokenizer, TFDistilBertForSequenceClassification
+    >>> model_name = "distilbert-base-uncased-finetuned-sst-2-english"
+    >>> model = TFDistilBertForSequenceClassification.from_pretrained(model_name)
+    >>> tokenizer = DistilBertTokenizer.from_pretrained(model_name)
 
 Customizing the model
 ^^^^^^^^^^^^^^^^^^^^^
@@ -345,18 +360,18 @@ Here we use the predefined vocabulary of DistilBERT (hence load the tokenizer wi
 instantiate the model from the configuration instead of using the
 :func:`~transformers.DistilBertForSequenceClassification.from_pretrained` method).
 
-::
+.. code-block::
 
-    ## PYTORCH CODE
-    from transformers import DistilBertConfig, DistilBertTokenizer, DistilBertForSequenceClassification
-    config = DistilBertConfig(n_heads=8, dim=512, hidden_dim=4*512)
-    tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
-    model = DistilBertForSequenceClassification(config)
-    ## TENSORFLOW CODE
-    from transformers import DistilBertConfig, DistilBertTokenizer, TFDistilBertForSequenceClassification
-    config = DistilBertConfig(n_heads=8, dim=512, hidden_dim=4*512)
-    tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
-    model = TFDistilBertForSequenceClassification(config)
+    >>> ## PYTORCH CODE
+    >>> from transformers import DistilBertConfig, DistilBertTokenizer, DistilBertForSequenceClassification
+    >>> config = DistilBertConfig(n_heads=8, dim=512, hidden_dim=4*512)
+    >>> tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
+    >>> model = DistilBertForSequenceClassification(config)
+    >>> ## TENSORFLOW CODE
+    >>> from transformers import DistilBertConfig, DistilBertTokenizer, TFDistilBertForSequenceClassification
+    >>> config = DistilBertConfig(n_heads=8, dim=512, hidden_dim=4*512)
+    >>> tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
+    >>> model = TFDistilBertForSequenceClassification(config)
 
 For something that only changes the head of the model (for instance, the number of labels), you can still use a
 pretrained model for the body. For instance, let's define a classifier for 10 different labels using a pretrained body.
@@ -364,15 +379,15 @@ We could create a configuration with all the default values and just change the
 can directly pass any argument a configuration would take to the :func:`from_pretrained` method and it will update the
 default configuration with it:
 
-::
-
-    ## PYTORCH CODE
-    from transformers import DistilBertConfig, DistilBertTokenizer, DistilBertForSequenceClassification
-    model_name = "distilbert-base-uncased"
-    model = DistilBertForSequenceClassification.from_pretrained(model_name, num_labels=10)
-    tokenizer = DistilBertTokenizer.from_pretrained(model_name)
-    ## TENSORFLOW CODE
-    from transformers import DistilBertConfig, DistilBertTokenizer, TFDistilBertForSequenceClassification
-    model_name = "distilbert-base-uncased"
-    model = TFDistilBertForSequenceClassification.from_pretrained(model_name, num_labels=10)
-    tokenizer = DistilBertTokenizer.from_pretrained(model_name)
+.. code-block::
+
+    >>> ## PYTORCH CODE
+    >>> from transformers import DistilBertConfig, DistilBertTokenizer, DistilBertForSequenceClassification
+    >>> model_name = "distilbert-base-uncased"
+    >>> model = DistilBertForSequenceClassification.from_pretrained(model_name, num_labels=10)
+    >>> tokenizer = DistilBertTokenizer.from_pretrained(model_name)
+    >>> ## TENSORFLOW CODE
+    >>> from transformers import DistilBertConfig, DistilBertTokenizer, TFDistilBertForSequenceClassification
+    >>> model_name = "distilbert-base-uncased"
+    >>> model = TFDistilBertForSequenceClassification.from_pretrained(model_name, num_labels=10)
+    >>> tokenizer = DistilBertTokenizer.from_pretrained(model_name)

setup.py

@@ -86,7 +86,7 @@ extras["all"] = extras["serving"] + ["tensorflow", "torch"]
 
 extras["testing"] = ["pytest", "pytest-xdist", "timeout-decorator", "psutil"]
 # sphinx-rtd-theme==0.5.0 introduced big changes in the style.
-extras["docs"] = ["recommonmark", "sphinx", "sphinx-markdown-tables", "sphinx-rtd-theme==0.4.3"]
+extras["docs"] = ["recommonmark", "sphinx", "sphinx-markdown-tables", "sphinx-rtd-theme==0.4.3", "sphinx-copybutton"]
 extras["quality"] = [
     "black",
     "isort @ git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort",

src/transformers/configuration_albert.py

@@ -81,22 +81,22 @@ class AlbertConfig(PretrainedConfig):
 
         Example::
 
-            from transformers import AlbertConfig, AlbertModel
-            # Initializing an ALBERT-xxlarge style configuration
-            albert_xxlarge_configuration = AlbertConfig()
-
-            # Initializing an ALBERT-base style configuration
-            albert_base_configuration = AlbertConfig(
-                hidden_size=768,
-                num_attention_heads=12,
-                intermediate_size=3072,
-            )
-
-            # Initializing a model from the ALBERT-base style configuration
-            model = AlbertModel(albert_xxlarge_configuration)
-
-            # Accessing the model configuration
-            configuration = model.config
+            >>> from transformers import AlbertConfig, AlbertModel
+            >>> # Initializing an ALBERT-xxlarge style configuration
+            >>> albert_xxlarge_configuration = AlbertConfig()
+
+            >>> # Initializing an ALBERT-base style configuration
+            >>> albert_base_configuration = AlbertConfig(
+            ...      hidden_size=768,
+            ...      num_attention_heads=12,
+            ...      intermediate_size=3072,
+            ...  )
+
+            >>> # Initializing a model from the ALBERT-base style configuration
+            >>> model = AlbertModel(albert_xxlarge_configuration)
+
+            >>> # Accessing the model configuration
+            >>> configuration = model.config
     """
 
     model_type = "albert"

src/transformers/configuration_bart.py

@@ -73,9 +73,13 @@ class BartConfig(PretrainedConfig):
     ):
         r"""
             :class:`~transformers.BartConfig` is the configuration class for `BartModel`.
-            Examples:
-                config = BartConfig.from_pretrained('bart-large')
-                model = BartModel(config)
+
+            Examples::
+
+                >>> from transformers import BartConfig, BartModel
+
+                >>> config = BartConfig.from_pretrained('facebook/bart-large')
+                >>> model = BartModel(config)
         """
         if "hidden_size" in common_kwargs:
             raise ValueError("hidden size is called d_model")

src/transformers/configuration_bert.py

@@ -95,16 +95,16 @@ class BertConfig(PretrainedConfig):
 
         Example::
 
-            from transformers import BertModel, BertConfig
+            >>> from transformers import BertModel, BertConfig
 
-            # Initializing a BERT bert-base-uncased style configuration
-            configuration = BertConfig()
+            >>> # Initializing a BERT bert-base-uncased style configuration
+            >>> configuration = BertConfig()
 
-            # Initializing a model from the bert-base-uncased style configuration
-            model = BertModel(configuration)
+            >>> # Initializing a model from the bert-base-uncased style configuration
+            >>> model = BertModel(configuration)
 
-            # Accessing the model configuration
-            configuration = model.config
+            >>> # Accessing the model configuration
+            >>> configuration = model.config
     """
     model_type = "bert"
 

src/transformers/configuration_ctrl.py

@@ -66,16 +66,16 @@ class CTRLConfig(PretrainedConfig):
 
         Example::
 
-            from transformers import CTRLModel, CTRLConfig
+            >>> from transformers import CTRLModel, CTRLConfig
 
-            # Initializing a CTRL configuration
-            configuration = CTRLConfig()
+            >>> # Initializing a CTRL configuration
+            >>> configuration = CTRLConfig()
 
-            # Initializing a model from the configuration
-            model = CTRLModel(configuration)
+            >>> # Initializing a model from the configuration
+            >>> model = CTRLModel(configuration)
 
-            # Accessing the model configuration
-            configuration = model.config
+            >>> # Accessing the model configuration
+            >>> configuration = model.config
     """
 
     model_type = "ctrl"

src/transformers/configuration_distilbert.py

@@ -80,16 +80,16 @@ class DistilBertConfig(PretrainedConfig):
 
         Example::
 
-            from transformers import DistilBertModel, DistilBertConfig
+            >>> from transformers import DistilBertModel, DistilBertConfig
 
-            # Initializing a DistilBERT configuration
-            configuration = DistilBertConfig()
+            >>> # Initializing a DistilBERT configuration
+            >>> configuration = DistilBertConfig()
 
-            # Initializing a model from the configuration
-            model = DistilBertModel(configuration)
+            >>> # Initializing a model from the configuration
+            >>> model = DistilBertModel(configuration)
 
-            # Accessing the model configuration
-            configuration = model.config
+            >>> # Accessing the model configuration
+            >>> configuration = model.config
     """
     model_type = "distilbert"
 

src/transformers/configuration_electra.py

@@ -101,16 +101,16 @@ class ElectraConfig(PretrainedConfig):
 
         Example::
 
-            from transformers import ElectraModel, ElectraConfig
+            >>> from transformers import ElectraModel, ElectraConfig
 
-            # Initializing a ELECTRA electra-base-uncased style configuration
-            configuration = ElectraConfig()
+            >>> # Initializing a ELECTRA electra-base-uncased style configuration
+            >>> configuration = ElectraConfig()
 
-            # Initializing a model from the electra-base-uncased style configuration
-            model = ElectraModel(configuration)
+            >>> # Initializing a model from the electra-base-uncased style configuration
+            >>> model = ElectraModel(configuration)
 
-            # Accessing the model configuration
-            configuration = model.config
+            >>> # Accessing the model configuration
+            >>> configuration = model.config
     """
     model_type = "electra"
 

src/transformers/configuration_encoder_decoder.py

@@ -42,20 +42,20 @@ class EncoderDecoderConfig(PretrainedConfig):
 
         Example::
 
-            from transformers import BertConfig, EncoderDecoderConfig, EncoderDecoderModel
+            >>> from transformers import BertConfig, EncoderDecoderConfig, EncoderDecoderModel
 
-            # Initializing a BERT bert-base-uncased style configuration
-            config_encoder = BertConfig()
-            config_decoder = BertConfig()
+            >>> # Initializing a BERT bert-base-uncased style configuration
+            >>> config_encoder = BertConfig()
+            >>> config_decoder = BertConfig()
 
-            config = EncoderDecoderConfig.from_encoder_decoder_configs(config_encoder, config_decoder)
+            >>> config = EncoderDecoderConfig.from_encoder_decoder_configs(config_encoder, config_decoder)
 
-            # Initializing a Bert2Bert model from the bert-base-uncased style configurations
-            model = EncoderDecoderModel(config=config)
+            >>> # Initializing a Bert2Bert model from the bert-base-uncased style configurations
+            >>> model = EncoderDecoderModel(config=config)
 
-            # Accessing the model configuration
-            config_encoder = model.config.encoder
-            config_decoder  = model.config.decoder
+            >>> # Accessing the model configuration
+            >>> config_encoder = model.config.encoder
+            >>> config_decoder  = model.config.decoder
     """
     model_type = "encoder_decoder"
 

src/transformers/configuration_gpt2.py

@@ -100,16 +100,16 @@ class GPT2Config(PretrainedConfig):
 
         Example::
 
-            from transformers import GPT2Model, GPT2Config
+            >>> from transformers import GPT2Model, GPT2Config
 
-            # Initializing a GPT2 configuration
-            configuration = GPT2Config()
+            >>> # Initializing a GPT2 configuration
+            >>> configuration = GPT2Config()
 
-            # Initializing a model from the configuration
-            model = GPT2Model(configuration)
+            >>> # Initializing a model from the configuration
+            >>> model = GPT2Model(configuration)
 
-            # Accessing the model configuration
-            configuration = model.config
+            >>> # Accessing the model configuration
+            >>> configuration = model.config
     """
 
     model_type = "gpt2"

src/transformers/configuration_longformer.py

@@ -49,16 +49,16 @@ class LongformerConfig(RobertaConfig):
 
         Example::
 
-            from transformers import LongformerConfig, LongformerModel
+            >>> from transformers import LongformerConfig, LongformerModel
 
-            # Initializing a Longformer configuration
-            configuration = LongformerConfig()
+            >>> # Initializing a Longformer configuration
+            >>> configuration = LongformerConfig()
 
-            # Initializing a model from the configuration
-            model = LongformerModel(configuration)
+            >>> # Initializing a model from the configuration
+            >>> model = LongformerModel(configuration)
 
-            # Accessing the model configuration
-            configuration = model.config
+            >>> # Accessing the model configuration
+            >>> configuration = model.config
     """
     model_type = "longformer"
 

src/transformers/configuration_mobilebert.py

@@ -85,16 +85,16 @@ class MobileBertConfig(PretrainedConfig):
 
         Example:
 
-            from transformers import MobileBertModel, MobileBertConfig
+            >>> from transformers import MobileBertModel, MobileBertConfig
 
-            # Initializing a MobileBERT configuration
-            configuration = MobileBertConfig()
+            >>> # Initializing a MobileBERT configuration
+            >>> configuration = MobileBertConfig()
 
-            # Initializing a model from the configuration above
-            model = MobileBertModel(configuration)
+            >>> # Initializing a model from the configuration above
+            >>> model = MobileBertModel(configuration)
 
-            # Accessing the model configuration
-            configuration = model.config
+            >>> # Accessing the model configuration
+            >>> configuration = model.config
 
         Attributes:
             pretrained_config_archive_map (Dict[str, str]):

src/transformers/configuration_openai.py

@@ -98,16 +98,16 @@ class OpenAIGPTConfig(PretrainedConfig):
 
         Example::
 
-            from transformers import OpenAIGPTConfig, OpenAIGPTModel
+            >>> from transformers import OpenAIGPTConfig, OpenAIGPTModel
 
-            # Initializing a GPT configuration
-            configuration = OpenAIGPTConfig()
+            >>> # Initializing a GPT configuration
+            >>> configuration = OpenAIGPTConfig()
 
-            # Initializing a model from the configuration
-            model = OpenAIGPTModel(configuration)
+            >>> # Initializing a model from the configuration
+            >>> model = OpenAIGPTModel(configuration)
 
-            # Accessing the model configuration
-            configuration = model.config
+            >>> # Accessing the model configuration
+            >>> configuration = model.config
     """
 
     model_type = "openai-gpt"