Model parallel documentation (#8741)

* Add parallelize methods to the .rst files

* Correct format

docs/source/model_doc/gpt2.rst

@@ -71,14 +71,14 @@ GPT2Model
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. autoclass:: transformers.GPT2Model
-    :members: forward
+    :members: forward, parallelize, deparallelize
 
 
 GPT2LMHeadModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. autoclass:: transformers.GPT2LMHeadModel
-    :members: forward
+    :members: forward, parallelize, deparallelize
 
 
 GPT2DoubleHeadsModel

docs/source/model_doc/t5.rst

@@ -99,14 +99,14 @@ T5Model
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. autoclass:: transformers.T5Model
-    :members: forward
+    :members: forward, parallelize, deparallelize
 
 
 T5ForConditionalGeneration
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. autoclass:: transformers.T5ForConditionalGeneration
-    :members: forward
+    :members: forward, parallelize, deparallelize
 
 
 TFT5Model

src/transformers/models/gpt2/modeling_gpt2.py

@@ -492,7 +492,8 @@ PARALLELIZE_DOCSTRING = r"""
                 - gpt2-xl: 48
 
     Example::
-        Here is an example of a device map on a machine with 4 GPUs using gpt2-xl, which has a total of 48 attention modules:
+
+            # Here is an example of a device map on a machine with 4 GPUs using gpt2-xl, which has a total of 48 attention modules:
             model = GPT2LMHeadModel.from_pretrained('gpt2-xl')
             device_map = {0: [0, 1, 2, 3, 4, 5, 6, 7, 8],
 
@@ -505,7 +506,8 @@ DEPARALLELIZE_DOCSTRING = r"""
     Moves the model to cpu from a model parallel state.
 
     Example::
-        On a 4 GPU machine with gpt2-large:
+
+        # On a 4 GPU machine with gpt2-large:
         model = GPT2LMHeadModel.from_pretrained('gpt2-large')
         device_map = {0: [0, 1, 2, 3, 4, 5, 6, 7],
 

src/transformers/models/t5/modeling_t5.py

@@ -196,7 +196,8 @@ PARALLELIZE_DOCSTRING = r"""
                 - t5-11b: 24
 
     Example::
-        Here is an example of a device map on a machine with 4 GPUs using t5-3b, which has a total of 24 attention modules:
+
+            # Here is an example of a device map on a machine with 4 GPUs using t5-3b, which has a total of 24 attention modules:
             model = T5ForConditionalGeneration.from_pretrained('t5-3b')
             device_map = {0: [0, 1, 2],
 
@@ -209,7 +210,8 @@ DEPARALLELIZE_DOCSTRING = r"""
     Moves the model to cpu from a model parallel state.
 
     Example::
-        On a 4 GPU machine with t5-3b:
+
+        # On a 4 GPU machine with t5-3b:
         model = T5ForConditionalGeneration.from_pretrained('t5-3b')
         device_map = {0: [0, 1, 2],