Skip to content

GitLab

Unverified Commit b4365e01 authored by J-shang's avatar J-shang Committed by GitHub
Browse files

[Doc] update results (#5105)

parent ecd08f8f
Hide whitespace changes
Inline Side-by-side
Showing with 125 additions and 82 deletions
docs/source/conf.py
View file @ b4365e01
...@@ -111,7 +111,8 @@ linkcheck_ignore = [ ...@@ -111,7 +111,8 @@ linkcheck_ignore = [
r'https://1drv\.ms/', # OneDrive (shortcut) r'https://1drv\.ms/', # OneDrive (shortcut)
r'https://onedrive\.live\.com/', # OneDrive r'https://onedrive\.live\.com/', # OneDrive
r'https://www\.openml\.org/', # OpenML r'https://www\.openml\.org/', # OpenML
r'https://ml\.informatik\.uni-freiburg\.de/' r'https://ml\.informatik\.uni-freiburg\.de/',
r'https://docs\.nvidia\.com/deeplearning/',
] ]
# Ignore all links located in release.rst # Ignore all links located in release.rst
......
docs/source/tutorials/pruning_bert_glue.ipynb
View file @ b4365e01
...@@ -177,7 +177,7 @@ ...@@ -177,7 +177,7 @@
"cell_type": "markdown", "cell_type": "markdown",
"metadata": {}, "metadata": {},
"source": [ "source": [
"## Result\nThe speedup is test on the entire validation dataset with batch size 32 on A100.\nWe test under two pytorch version and found the latency varying widely.\n\nSetting 1: pytorch 1.12.1\n\nSetting 2: pytorch 1.10.0\n\n.. list-table:: Prune Bert-base-uncased on MNLI\n :header-rows: 1\n :widths: auto\n\n * - Attention Pruning Method\n - FFN Pruning Method\n - Total Sparsity\n - Accuracy\n - Acc. Drop\n - Speedup (S1)\n - Speedup (S2)\n * -\n -\n - 0%\n - 84.73 / 84.63\n - +0.0 / +0.0\n - 12.56s (x1.00)\n - 4.05s (x1.00)\n * - `movement-pruner` (soft, sparsity=0.1, regular_scale=5)\n - `taylor-fo-weight-pruner`\n - 51.39%\n - 84.25 / 84.96\n - -0.48 / +0.33\n - 6.85s (x1.83)\n - 2.7s (x1.50)\n * - `movement-pruner` (soft, sparsity=0.1, regular_scale=10)\n - `taylor-fo-weight-pruner`\n - 66.67%\n - 83.98 / 83.75\n - -0.75 / -0.88\n - 4.73s (x2.66)\n - 2.16s (x1.86)\n * - `movement-pruner` (soft, sparsity=0.1, regular_scale=20)\n - `taylor-fo-weight-pruner`\n - 77.78%\n - 83.02 / 83.06\n - -1.71 / -1.57\n - 3.35s (x3.75)\n - 1.72s (x2.35)\n * - `movement-pruner` (soft, sparsity=0.1, regular_scale=30)\n - `taylor-fo-weight-pruner`\n - 87.04%\n - 81.24 / 80.99\n - -3.49 / -3.64\n - 2.19s (x5.74)\n - 1.31s (x3.09)\n\n" "## Result\nThe speedup is test on the entire validation dataset with batch size 128 on A100.\nWe test under two pytorch version and found the latency varying widely.\n\nSetting 1: pytorch 1.12.1\n\nSetting 2: pytorch 1.10.0\n\n.. list-table:: Prune Bert-base-uncased on MNLI\n :header-rows: 1\n :widths: auto\n\n * - Attention Pruning Method\n - FFN Pruning Method\n - Total Sparsity\n - Accuracy\n - Acc. Drop\n - Speedup (S1)\n - Speedup (S2)\n * -\n -\n - 85.1M (-0.0%)\n - 84.85 / 85.28\n - +0.0 / +0.0\n - 25.60s (x1.00)\n - 8.10s (x1.00)\n * - `movement-pruner` (soft, sparsity=0.1, regular_scale=1)\n - `taylor-fo-weight-pruner`\n - 54.1M (-36.43%)\n - 85.38 / 85.41\n - +0.53 / +0.13\n - 17.93s (x1.43)\n - 7.22s (x1.12)\n * - `movement-pruner` (soft, sparsity=0.1, regular_scale=5)\n - `taylor-fo-weight-pruner`\n - 37.1M (-56.40%)\n - 84.73 / 85.12\n - -0.12 / -0.16\n - 12.83s (x2.00)\n - 5.61s (x1.44)\n * - `movement-pruner` (soft, sparsity=0.1, regular_scale=10)\n - `taylor-fo-weight-pruner`\n - 24.1M (-71.68%)\n - 84.14 / 84.78\n - -0.71 / -0.50\n - 8.93s (x2.87)\n - 4.55s (x1.78)\n * - `movement-pruner` (soft, sparsity=0.1, regular_scale=20)\n - `taylor-fo-weight-pruner`\n - 14.3M (-83.20%)\n - 83.26 / 82.96\n - -1.59 / -2.32\n - 5.98s (x4.28)\n - 3.56s (x2.28)\n * - `movement-pruner` (soft, sparsity=0.1, regular_scale=30)\n - `taylor-fo-weight-pruner`\n - 9.9M (-88.37%)\n - 82.22 / 82.19\n - -2.63 / -3.09\n - 4.36s (x5.88)\n - 3.12s (x2.60)\n * - `movement-pruner` (soft, sparsity=0.1, regular_scale=40)\n - `taylor-fo-weight-pruner`\n - 8.8M (-89.66%)\n - 81.64 / 82.39\n - -3.21 / -2.89\n - 3.88s (x6.60)\n - 2.81s (x2.88)\n\n"
] ]
} }
], ],
......
docs/source/tutorials/pruning_bert_glue.py
View file @ b4365e01
...@@ -537,7 +537,7 @@ for current_epoch in range(total_epochs): ...@@ -537,7 +537,7 @@ for current_epoch in range(total_epochs):
# %% # %%
# Result # Result
# ------ # ------
# The speedup is test on the entire validation dataset with batch size 32 on A100. # The speedup is test on the entire validation dataset with batch size 128 on A100.
# We test under two pytorch version and found the latency varying widely. # We test under two pytorch version and found the latency varying widely.
# #
# Setting 1: pytorch 1.12.1 # Setting 1: pytorch 1.12.1
...@@ -557,36 +557,50 @@ for current_epoch in range(total_epochs): ...@@ -557,36 +557,50 @@ for current_epoch in range(total_epochs):
# - Speedup (S2) # - Speedup (S2)
# * - # * -
# - # -
# - 0% # - 85.1M (-0.0%)
# - 84.73 / 84.63 # - 84.85 / 85.28
# - +0.0 / +0.0 # - +0.0 / +0.0
# - 12.56s (x1.00) # - 25.60s (x1.00)
# - 4.05s (x1.00) # - 8.10s (x1.00)
# * - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=1)
# - :ref:`taylor-fo-weight-pruner`
# - 54.1M (-36.43%)
# - 85.38 / 85.41
# - +0.53 / +0.13
# - 17.93s (x1.43)
# - 7.22s (x1.12)
# * - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=5) # * - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=5)
# - :ref:`taylor-fo-weight-pruner` # - :ref:`taylor-fo-weight-pruner`
# - 51.39% # - 37.1M (-56.40%)
# - 84.25 / 84.96 # - 84.73 / 85.12
# - -0.48 / +0.33 # - -0.12 / -0.16
# - 6.85s (x1.83) # - 12.83s (x2.00)
# - 2.7s (x1.50) # - 5.61s (x1.44)
# * - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=10) # * - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=10)
# - :ref:`taylor-fo-weight-pruner` # - :ref:`taylor-fo-weight-pruner`
# - 66.67% # - 24.1M (-71.68%)
# - 83.98 / 83.75 # - 84.14 / 84.78
# - -0.75 / -0.88 # - -0.71 / -0.50
# - 4.73s (x2.66) # - 8.93s (x2.87)
# - 2.16s (x1.86) # - 4.55s (x1.78)
# * - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=20) # * - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=20)
# - :ref:`taylor-fo-weight-pruner` # - :ref:`taylor-fo-weight-pruner`
# - 77.78% # - 14.3M (-83.20%)
# - 83.02 / 83.06 # - 83.26 / 82.96
# - -1.71 / -1.57 # - -1.59 / -2.32
# - 3.35s (x3.75) # - 5.98s (x4.28)
# - 1.72s (x2.35) # - 3.56s (x2.28)
# * - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=30) # * - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=30)
# - :ref:`taylor-fo-weight-pruner` # - :ref:`taylor-fo-weight-pruner`
# - 87.04% # - 9.9M (-88.37%)
# - 81.24 / 80.99 # - 82.22 / 82.19
# - -3.49 / -3.64 # - -2.63 / -3.09
# - 2.19s (x5.74) # - 4.36s (x5.88)
# - 1.31s (x3.09) # - 3.12s (x2.60)
# * - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=40)
# - :ref:`taylor-fo-weight-pruner`
# - 8.8M (-89.66%)
# - 81.64 / 82.39
# - -3.21 / -2.89
# - 3.88s (x6.60)
# - 2.81s (x2.88)
docs/source/tutorials/pruning_bert_glue.py.md5
View file @ b4365e01
4935f5727dd073c91bcfab8b9f0676d7 d3191675dd9427c6906f2bd3929ee382
\ No newline at end of file \ No newline at end of file
docs/source/tutorials/pruning_bert_glue.rst
View file @ b4365e01
...@@ -643,11 +643,11 @@ NNI will support per-step-pruning-schedule in the future, then can use an pruner ...@@ -643,11 +643,11 @@ NNI will support per-step-pruning-schedule in the future, then can use an pruner
.. GENERATED FROM PYTHON SOURCE LINES 538-593 .. GENERATED FROM PYTHON SOURCE LINES 538-607
Result Result
------ ------
The speedup is test on the entire validation dataset with batch size 32 on A100. The speedup is test on the entire validation dataset with batch size 128 on A100.
We test under two pytorch version and found the latency varying widely. We test under two pytorch version and found the latency varying widely.
Setting 1: pytorch 1.12.1 Setting 1: pytorch 1.12.1
...@@ -667,44 +667,58 @@ Setting 2: pytorch 1.10.0 ...@@ -667,44 +667,58 @@ Setting 2: pytorch 1.10.0
- Speedup (S2) - Speedup (S2)
* - * -
- -
- 0% - 85.1M (-0.0%)
- 84.73 / 84.63 - 84.85 / 85.28
- +0.0 / +0.0 - +0.0 / +0.0
- 12.56s (x1.00) - 25.60s (x1.00)
- 4.05s (x1.00) - 8.10s (x1.00)
* - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=1)
- :ref:`taylor-fo-weight-pruner`
- 54.1M (-36.43%)
- 85.38 / 85.41
- +0.53 / +0.13
- 17.93s (x1.43)
- 7.22s (x1.12)
* - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=5) * - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=5)
- :ref:`taylor-fo-weight-pruner` - :ref:`taylor-fo-weight-pruner`
- 51.39% - 37.1M (-56.40%)
- 84.25 / 84.96 - 84.73 / 85.12
- -0.48 / +0.33 - -0.12 / -0.16
- 6.85s (x1.83) - 12.83s (x2.00)
- 2.7s (x1.50) - 5.61s (x1.44)
* - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=10) * - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=10)
- :ref:`taylor-fo-weight-pruner` - :ref:`taylor-fo-weight-pruner`
- 66.67% - 24.1M (-71.68%)
- 83.98 / 83.75 - 84.14 / 84.78
- -0.75 / -0.88 - -0.71 / -0.50
- 4.73s (x2.66) - 8.93s (x2.87)
- 2.16s (x1.86) - 4.55s (x1.78)
* - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=20) * - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=20)
- :ref:`taylor-fo-weight-pruner` - :ref:`taylor-fo-weight-pruner`
- 77.78% - 14.3M (-83.20%)
- 83.02 / 83.06 - 83.26 / 82.96
- -1.71 / -1.57 - -1.59 / -2.32
- 3.35s (x3.75) - 5.98s (x4.28)
- 1.72s (x2.35) - 3.56s (x2.28)
* - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=30) * - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=30)
- :ref:`taylor-fo-weight-pruner` - :ref:`taylor-fo-weight-pruner`
- 87.04% - 9.9M (-88.37%)
- 81.24 / 80.99 - 82.22 / 82.19
- -3.49 / -3.64 - -2.63 / -3.09
- 2.19s (x5.74) - 4.36s (x5.88)
- 1.31s (x3.09) - 3.12s (x2.60)
* - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=40)
- :ref:`taylor-fo-weight-pruner`
- 8.8M (-89.66%)
- 81.64 / 82.39
- -3.21 / -2.89
- 3.88s (x6.60)
- 2.81s (x2.88)
.. rst-class:: sphx-glr-timing .. rst-class:: sphx-glr-timing
**Total running time of the script:** ( 0 minutes 41.637 seconds) **Total running time of the script:** ( 0 minutes 20.822 seconds)
.. _sphx_glr_download_tutorials_pruning_bert_glue.py: .. _sphx_glr_download_tutorials_pruning_bert_glue.py:
......
docs/source/tutorials/sg_execution_times.rst
View file @ b4365e01
...@@ -5,10 +5,10 @@ ...@@ -5,10 +5,10 @@
Computation times Computation times
================= =================
**01:51.710** total execution time for **tutorials** files: **00:20.822** total execution time for **tutorials** files:
+-----------------------------------------------------------------------------------------------------+-----------+--------+ +-----------------------------------------------------------------------------------------------------+-----------+--------+
| :ref:`sphx_glr_tutorials_pruning_bert_glue.py` (``pruning_bert_glue.py``) | 00:00.000 | 0.0 MB | | :ref:`sphx_glr_tutorials_pruning_bert_glue.py` (``pruning_bert_glue.py``) | 00:20.822 | 0.0 MB |
+-----------------------------------------------------------------------------------------------------+-----------+--------+ +-----------------------------------------------------------------------------------------------------+-----------+--------+
| :ref:`sphx_glr_tutorials_darts.py` (``darts.py``) | 01:51.710 | 0.0 MB | | :ref:`sphx_glr_tutorials_darts.py` (``darts.py``) | 01:51.710 | 0.0 MB |
+-----------------------------------------------------------------------------------------------------+-----------+--------+ +-----------------------------------------------------------------------------------------------------+-----------+--------+
......
examples/tutorials/pruning_bert_glue.py
View file @ b4365e01
...@@ -537,7 +537,7 @@ for current_epoch in range(total_epochs): ...@@ -537,7 +537,7 @@ for current_epoch in range(total_epochs):
# %% # %%
# Result # Result
# ------ # ------
# The speedup is test on the entire validation dataset with batch size 32 on A100. # The speedup is test on the entire validation dataset with batch size 128 on A100.
# We test under two pytorch version and found the latency varying widely. # We test under two pytorch version and found the latency varying widely.
# #
# Setting 1: pytorch 1.12.1 # Setting 1: pytorch 1.12.1
...@@ -557,36 +557,50 @@ for current_epoch in range(total_epochs): ...@@ -557,36 +557,50 @@ for current_epoch in range(total_epochs):
# - Speedup (S2) # - Speedup (S2)
# * - # * -
# - # -
# - 0% # - 85.1M (-0.0%)
# - 84.73 / 84.63 # - 84.85 / 85.28
# - +0.0 / +0.0 # - +0.0 / +0.0
# - 12.56s (x1.00) # - 25.60s (x1.00)
# - 4.05s (x1.00) # - 8.10s (x1.00)
# * - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=1)
# - :ref:`taylor-fo-weight-pruner`
# - 54.1M (-36.43%)
# - 85.38 / 85.41
# - +0.53 / +0.13
# - 17.93s (x1.43)
# - 7.22s (x1.12)
# * - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=5) # * - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=5)
# - :ref:`taylor-fo-weight-pruner` # - :ref:`taylor-fo-weight-pruner`
# - 51.39% # - 37.1M (-56.40%)
# - 84.25 / 84.96 # - 84.73 / 85.12
# - -0.48 / +0.33 # - -0.12 / -0.16
# - 6.85s (x1.83) # - 12.83s (x2.00)
# - 2.7s (x1.50) # - 5.61s (x1.44)
# * - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=10) # * - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=10)
# - :ref:`taylor-fo-weight-pruner` # - :ref:`taylor-fo-weight-pruner`
# - 66.67% # - 24.1M (-71.68%)
# - 83.98 / 83.75 # - 84.14 / 84.78
# - -0.75 / -0.88 # - -0.71 / -0.50
# - 4.73s (x2.66) # - 8.93s (x2.87)
# - 2.16s (x1.86) # - 4.55s (x1.78)
# * - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=20) # * - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=20)
# - :ref:`taylor-fo-weight-pruner` # - :ref:`taylor-fo-weight-pruner`
# - 77.78% # - 14.3M (-83.20%)
# - 83.02 / 83.06 # - 83.26 / 82.96
# - -1.71 / -1.57 # - -1.59 / -2.32
# - 3.35s (x3.75) # - 5.98s (x4.28)
# - 1.72s (x2.35) # - 3.56s (x2.28)
# * - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=30) # * - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=30)
# - :ref:`taylor-fo-weight-pruner` # - :ref:`taylor-fo-weight-pruner`
# - 87.04% # - 9.9M (-88.37%)
# - 81.24 / 80.99 # - 82.22 / 82.19
# - -3.49 / -3.64 # - -2.63 / -3.09
# - 2.19s (x5.74) # - 4.36s (x5.88)
# - 1.31s (x3.09) # - 3.12s (x2.60)
# * - :ref:`movement-pruner` (soft, sparsity=0.1, regular_scale=40)
# - :ref:`taylor-fo-weight-pruner`
# - 8.8M (-89.66%)
# - 81.64 / 82.39
# - -3.21 / -2.89
# - 3.88s (x6.60)
# - 2.81s (x2.88)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment