Skip to content

GitLab

Commit 2b05e121 authored by yuguo's avatar yuguo
Browse files

Merge commit 'a69692ac' of... 

Merge commit 'a69692ac' of https://github.com/NVIDIA/TransformerEngine
parents 0fd441c2 a69692ac
Expand all Show whitespace changes
Inline Side-by-side
Showing with 157 additions and 52 deletions
docs/debug/api.rst 0 → 100644
View file @ 2b05e121
..
Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
See LICENSE for license information.
API
============
.. toctree::
:caption: Precision debug tools API
3_api_debug_setup.rst
3_api_features.rst
3_api_te_calls.rst
\ No newline at end of file
docs/debug/img/api_calls1.svg 0 → 100644
View file @ 2b05e121
This diff is collapsed.
docs/debug/img/api_calls2.svg 0 → 100644
View file @ 2b05e121
<svg width="4235" height="2342" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xml:space="preserve" overflow="hidden"><g transform="translate(-41 -119)"><g><rect x="46.4999" y="1576.5" width="1564" height="734" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FFFFFF" fill-opacity="1"/><rect x="630.5" y="125.5" width="580" height="151" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FFFFFF" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="73" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 786.823 225)">Tensor A</text><rect x="303.5" y="337.5" width="1234" height="106" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#DCEAF7" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="73" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 684.099 414)">inspect_tensor</text><rect x="1258.5" y="596.5" width="617" height="106" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FFFFFF" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="73" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1440.36 673)">fp8 </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="73" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1557.81 673)">cast</text><rect x="114.5" y="596.5" width="683" height="106" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#DCEAF7" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="73" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 227.611 673)">modify_tensor</text><rect x="303.5" y="826.5" width="1234" height="106" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#DCEAF7" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="73" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 460.753 903)">inspect_tensor_postquantize</text><rect x="1583.5" y="1123.5" width="1234" height="106" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FFFFFF" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="73" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2095.73 1200)">GEMM</text><rect x="1583.5" y="1310.5" width="1234" height="106" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#DCEAF7" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="73" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1963.85 1387)">inspect_tensor</text><rect x="1859.5" y="1499.5" width="682" height="106" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#DCEAF7" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="73" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1972.18 1576)">modify_tensor</text><rect x="115.5" y="1956.5" width="1402" height="106" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#DCEAF7" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="73" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 434.13 2033)">inspect_tensor_enabled</text><rect x="115.5" y="2103.5" width="1402" height="106" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#DCEAF7" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="73" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 210.785 2180)">inspect_tensor_postquantize_enabled</text><rect x="115.5" y="1660.5" width="1402" height="106" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#DCEAF7" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="73" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 502.961 1737)">fp8_gemm_enabled</text><rect x="115.5" y="1808.5" width="1402" height="106" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#DCEAF7" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="73" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 442.461 1885)">modify_tensor_enabled</text><path d="M1.07643-3.26461 444.129 142.822 441.977 149.351-1.07643 3.26461ZM443.006 131.593 464.817 153.263 434.395 157.71Z" fill="#000000" fill-rule="nonzero" fill-opacity="1" transform="matrix(-1 0 0 1 920.317 443.5)"/><path d="M921.293 440.155 1545.58 588.133 1543.99 594.822 919.707 446.845ZM1543.5 577.041 1567.09 596.763 1537.16 603.8Z" fill="#000000" fill-rule="nonzero" fill-opacity="1"/><path d="M923.938 443.5 923.938 803.572 917.063 803.572 917.063 443.5ZM934.25 798.988 920.5 826.488 906.75 798.988Z" fill="#000000" fill-rule="nonzero" fill-opacity="1"/><path d="M456.384 699.178 899.056 817.032 897.288 823.676 454.616 705.822ZM897.281 805.888 920.317 826.25 890.206 832.462Z" fill="#000000" fill-rule="nonzero" fill-opacity="1"/><path d="M0.646175-3.37622 624.723 116.066 623.431 122.818-0.646175 3.37622ZM622.16 105.076 646.585 123.75 616.991 132.085Z" fill="#000000" fill-rule="nonzero" fill-opacity="1" transform="matrix(-1 0 0 1 1567.09 702.5)"/><rect x="2945.5" y="125.5" width="579" height="151" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FFFFFF" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="73" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 3100.27 225)">Tensor B</text><rect x="2617.5" y="337.5" width="1234" height="106" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#DCEAF7" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="73" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2998.12 414)">inspect_tensor</text><rect x="3572.5" y="596.5" width="617" height="106" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FFFFFF" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="73" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 3754.39 673)">fp8 </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="73" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 3871.84 673)">cast</text><rect x="2428.5" y="596.5" width="683" height="106" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#DCEAF7" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="73" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2541.64 673)">modify_tensor</text><rect x="2617.5" y="826.5" width="1234" height="106" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#DCEAF7" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="73" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2774.78 903)">inspect_tensor_postquantize</text><path d="M1.07643-3.26461 444.129 142.822 441.976 149.351-1.07643 3.26461ZM443.006 131.593 464.817 153.263 434.394 157.71Z" fill="#000000" fill-rule="nonzero" fill-opacity="1" transform="matrix(-1 0 0 1 3234.32 443.5)"/><path d="M3235.29 440.155 3859.58 588.133 3857.99 594.822 3233.71 446.845ZM3857.5 577.041 3881.09 596.763 3851.16 603.8Z" fill="#000000" fill-rule="nonzero" fill-opacity="1"/><path d="M3237.94 443.5 3237.94 803.572 3231.06 803.572 3231.06 443.5ZM3248.25 798.988 3234.5 826.488 3220.75 798.988Z" fill="#000000" fill-rule="nonzero" fill-opacity="1"/><path d="M2770.38 699.178 3213.06 817.032 3211.29 823.676 2768.62 705.822ZM3211.28 805.888 3234.32 826.25 3204.21 832.462Z" fill="#000000" fill-rule="nonzero" fill-opacity="1"/><path d="M0.646175-3.37622 624.723 116.066 623.431 122.818-0.646175 3.37622ZM622.16 105.076 646.585 123.75 616.991 132.085Z" fill="#000000" fill-rule="nonzero" fill-opacity="1" transform="matrix(-1 0 0 1 3881.09 702.5)"/><path d="M921.009 929.1 2178.11 1117.2 2177.09 1124 919.991 935.9ZM2175.09 1106.33 2200.26 1123.99 2171.02 1133.52Z" fill="#000000" fill-rule="nonzero" fill-opacity="1"/><path d="M0.625813-3.38005 1012.36 183.941 1011.11 190.702-0.625813 3.38005ZM1009.73 172.967 1034.27 191.493 1004.72 200.007Z" fill="#000000" fill-rule="nonzero" fill-opacity="1" transform="matrix(-1 0 0 1 3234.77 932.5)"/><path d="M3.4375-1.54131e-05 3.43776 57.5713-3.43724 57.5714-3.4375 1.54131e-05ZM13.7502 52.988 0.000360892 80.488-13.7498 52.9881Z" fill="#000000" fill-rule="nonzero" fill-opacity="1" transform="matrix(-1 0 0 1 2200.5 1229.5)"/><path d="M3.4375-1.54131e-05 3.43776 57.5713-3.43724 57.5714-3.4375 1.54131e-05ZM13.7502 52.988 0.000360892 80.488-13.7498 52.9881Z" fill="#000000" fill-rule="nonzero" fill-opacity="1" transform="matrix(-1 0 0 1 2200.5 1418.5)"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="73" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 623.083 2394)">Routing </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="73" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 882.041 2394)">calls</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="73" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 3298.55 1286)">GEMM </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="73" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 3536.88 1286)">calls</text><path d="M923.938 276.5 923.938 314.619 917.063 314.62 917.063 276.5ZM934.25 310.036 920.5 337.536 906.75 310.036Z" fill="#000000" fill-rule="nonzero" fill-opacity="1"/><path d="M3237.94 276.5 3237.94 314.619 3231.06 314.62 3231.06 276.5ZM3248.25 310.036 3234.5 337.536 3220.75 310.036Z" fill="#000000" fill-rule="nonzero" fill-opacity="1"/></g></g></svg>
\ No newline at end of file
docs/debug/img/fake_quant.svg 0 → 100644
View file @ 2b05e121
<svg width="4111" height="1434" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xml:space="preserve" overflow="hidden"><g transform="translate(-91 -426)"><g><rect x="986" y="1109" width="330" height="221" fill="#DCEAF7" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="60" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1102.43 1203)">FP8 </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="60" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1066.64 1274)">GEMM</text><rect x="941" y="429" width="420" height="334" fill="#DCEAF7" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="60" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1085.53 580)">BF16</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="60" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1066.92 651)">weight</text><rect x="95.9999" y="1052" width="420" height="334" fill="#DCEAF7" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="60" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 239.829 1203)">BF16</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="60" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 239.256 1274)">input</text><rect x="600" y="1109" width="305" height="221" fill="#D9F2D0" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="60" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 703.883 1203)">FP8</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="60" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 686.397 1274)">input</text><rect x="999" y="831" width="305" height="221" fill="#DCEAF7" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="60" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1102.44 925)">FP8</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="60" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1066.92 996)">weight</text><rect x="1510" y="1052" width="420" height="334" fill="#DCEAF7" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="60" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1653.85 1203)">BF16</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="60" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1592.91 1274)">activation</text><path d="M1154.94 763.5 1154.94 808.797 1148.06 808.797 1148.06 763.5ZM1165.25 804.214 1151.5 831.714 1137.75 804.214Z" fill="#156082" fill-rule="nonzero" fill-opacity="1"/><path d="M1154.94 1046.5 1154.94 1091.8 1148.06 1091.8 1148.06 1046.5ZM1165.25 1087.21 1151.5 1114.71 1137.75 1087.21Z" fill="#156082" fill-rule="nonzero" fill-opacity="1"/><path d="M1.46568e-05-3.4375 61.7245-3.43724 61.7245 3.43776-1.46568e-05 3.4375ZM57.1412-13.7498 84.6412 0.000360892 57.1411 13.7502Z" fill="#156082" fill-rule="nonzero" fill-opacity="1" transform="matrix(1 0 0 -1 516.5 1219.5)"/><path d="M1.46568e-05-3.4375 61.7245-3.43724 61.7245 3.43776-1.46568e-05 3.4375ZM57.1412-13.7498 84.6412 0.000360892 57.1411 13.7502Z" fill="#156082" fill-rule="nonzero" fill-opacity="1" transform="matrix(1 0 0 -1 904.5 1219.5)"/><path d="M1316.5 1216.06 1486.89 1216.06 1486.89 1222.94 1316.5 1222.94ZM1482.31 1205.75 1509.81 1219.5 1482.31 1233.25Z" fill="#156082" fill-rule="nonzero" fill-opacity="1"/><rect x="3270" y="1052" width="332" height="334" fill="#DCEAF7" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="60" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 3370.09 1203)">BF16 </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="60" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 3351.19 1274)">GEMM</text><rect x="3226" y="429" width="420" height="334" fill="#DCEAF7" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="60" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 3370.08 580)">BF16</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="60" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 3351.48 651)">weight</text><rect x="2228" y="1052" width="420" height="334" fill="#DCEAF7" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="60" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2371.88 1203)">BF16</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="60" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2371.31 1274)">input</text><rect x="3780" y="1052" width="420" height="334" fill="#DCEAF7" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="60" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 3924.02 1203)">BF16</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="60" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 3863.09 1274)">activation</text><path d="M3439.94 763.5 3439.94 1029.51 3433.06 1029.51 3433.06 763.5ZM3450.25 1024.93 3436.5 1052.43 3422.75 1024.93Z" fill="#156082" fill-rule="nonzero" fill-opacity="1"/><path d="M2648.5 1216.06 2736.92 1216.06 2736.92 1222.94 2648.5 1222.94ZM2732.33 1205.75 2759.83 1219.5 2732.33 1233.25Z" fill="#156082" fill-rule="nonzero" fill-opacity="1"/><path d="M1.46568e-05-3.4375 61.7245-3.43724 61.7245 3.43776-1.46568e-05 3.4375ZM57.1412-13.7498 84.6412 0.000360892 57.1411 13.7502Z" fill="#156082" fill-rule="nonzero" fill-opacity="1" transform="matrix(1 0 0 -1 3185.5 1219.5)"/><path d="M3602.5 1216.06 3757.44 1216.06 3757.44 1222.94 3602.5 1222.94ZM3752.86 1205.75 3780.36 1219.5 3752.86 1233.25Z" fill="#156082" fill-rule="nonzero" fill-opacity="1"/><rect x="2759" y="1052" width="420" height="334" fill="#D9F2D0" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="60" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2903.22 1142)">BF16 </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="60" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2902.07 1213)">Input</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="50" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2809.26 1277)">fake quantized</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="50" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2900.65 1337)">to FP8</text><rect x="1628" y="1600" width="305" height="220" fill="#D9F2D0" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="60" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1731.92 1693)">FP8</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="60" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1714.44 1764)">input</text><rect x="2228" y="1523" width="420" height="334" fill="#D9F2D0" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="60" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2371.88 1613)">BF16 </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="60" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2370.74 1684)">Input</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="50" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2277.92 1748)">fake quantized</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="50" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2369.31 1808)">to FP8</text><path d="M2020.39 1673.54 2155.61 1673.54 2155.61 1702.71 2020.39 1702.71ZM2020.39 1717.29 2155.61 1717.29 2155.61 1746.46 2020.39 1746.46Z" fill="#000000" fill-rule="evenodd" fill-opacity="1"/></g></g></svg>
\ No newline at end of file
docs/debug/img/introduction.svg 0 → 100644
View file @ 2b05e121
This diff is collapsed.
docs/debug/img/names.svg 0 → 100644
View file @ 2b05e121
<svg width="2622" height="2062" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xml:space="preserve" overflow="hidden"><g transform="translate(-948 -183)"><g><rect x="956.5" y="193.5" width="1971" height="2048" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#F2F2F2" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="NVIDIA Sans,NVIDIA Sans_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="83" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1010.52 287)">Transformer Layer with name </text><text fill="#000000" fill-opacity="1" font-family="NVIDIA Sans,NVIDIA Sans_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="700" font-stretch="normal" font-size="83" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2161.9 287)">transformer_layer</text><rect x="1229.5" y="555.5" width="1425" height="653" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#F2F2F2" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="NVIDIA Sans,NVIDIA Sans_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="700" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1519.58 630)">transformer_layer.self_attn</text><rect x="1475.5" y="988.5" width="933" height="95.9999" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#F2F2F2" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="700" font-stretch="normal" font-size="46" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1607.42 1046)">transformer_layer.self_attn.proj</text><rect x="1308.5" y="737.5" width="1267" height="97.0003" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#F2F2F2" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="700" font-stretch="normal" font-size="46" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1423.51 796)">transformer_layer.self_attn.layernorm_linear_qkv</text><rect x="1364.5" y="1404.5" width="1155" height="542" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#F2F2F2" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="NVIDIA Sans,NVIDIA Sans_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="700" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1432.23 1480)">transformer_layer.layernorm_mlp</text><rect x="1485.5" y="1576.5" width="913" height="90.0001" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#F2F2F2" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="NVIDIA Sans,NVIDIA Sans_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="700" font-stretch="normal" font-size="46" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1530.65 1635)">transformer_layer.layernorm_mlp.fc1</text><rect x="1485.5" y="1748.5" width="913" height="89.9998" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#F2F2F2" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="NVIDIA Sans,NVIDIA Sans_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="700" font-stretch="normal" font-size="46" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1530.65 1807)">transformer_layer.layernorm_mlp.fc2</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="83" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 3208.71 811)">1 </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="83" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 3269.44 811)">Linear</text><path d="M2574.5 787.5 3176.64 787.5" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="round" stroke-miterlimit="10" stroke-dasharray="27.5 20.625" stroke-opacity="1" fill="none" fill-rule="evenodd"/><path d="M0 0 767.947 3.92362" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="round" stroke-miterlimit="10" stroke-dasharray="27.5 20.625" stroke-opacity="1" fill="none" fill-rule="evenodd" transform="matrix(1 0 0 -1 2408.5 1036.42)"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="83" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 3208.71 1058)">1 </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="83" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 3269.44 1058)">Linear</text><path d="M0 0 767.947 3.92362" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="round" stroke-miterlimit="10" stroke-dasharray="27.5 20.625" stroke-opacity="1" fill="none" fill-rule="evenodd" transform="matrix(1 0 0 -1 2398.5 1624.42)"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="83" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 3198.53 1646)">1 </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="83" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 3259.26 1646)">Linear</text><path d="M0 0 767.947 3.92362" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="round" stroke-miterlimit="10" stroke-dasharray="27.5 20.625" stroke-opacity="1" fill="none" fill-rule="evenodd" transform="matrix(1 0 0 -1 2398.5 1796.42)"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="83" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 3199.07 1818)">1 </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="83" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 3259.8 1818)">Linear</text></g></g></svg>
\ No newline at end of file
docs/debug/img/pipeline_logging.svg 0 → 100644
View file @ 2b05e121
<svg width="3956" height="1347" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xml:space="preserve" overflow="hidden"><g transform="translate(-224 -527)"><g><rect x="652.5" y="567.5" width="497" height="577" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="83" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 772.973 660)">Node 1</text><rect x="1262.5" y="567.5" width="496" height="577" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="83" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1382.84 660)">Node 2</text><rect x="1871.5" y="567.5" width="497" height="577" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="83" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1992.71 660)">Node 3</text><rect x="2481.5" y="567.5" width="497" height="577" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="83" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2602.58 660)">Node 4</text><rect x="652.5" y="1238.5" width="497" height="577" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="83" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 772.973 1330)">Node 5</text><rect x="1262.5" y="1238.5" width="496" height="577" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="83" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1382.84 1330)">Node 6</text><rect x="1871.5" y="1238.5" width="497" height="577" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="83" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1992.71 1330)">Node 7</text><rect x="2481.5" y="1238.5" width="497" height="577" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="83" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2602.58 1330)">Node 8</text><rect x="3243.5" y="1117.5" width="932" height="241" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="83" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 3404.16 1264)">TensorBoard logs</text><rect x="2528.5" y="1486.5" width="403" height="110" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="83" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2572.13 1568)">tb_writer</text><rect x="2528.5" y="853.5" width="403" height="111" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="83" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2572.13 935)">tb_writer</text><path d="M2934 906.136 3230.16 1218.8 3225.17 1223.53 2929 910.864ZM3234.5 1208.39 3243.43 1237.81 3214.53 1227.3Z" fill="#000000" fill-rule="nonzero" fill-opacity="1"/><path d="M2.39725-2.46366 297.902 285.075 293.107 290.003-2.39725 2.46366ZM301.808 274.488 311.929 303.52 282.63 294.197Z" fill="#000000" fill-rule="nonzero" fill-opacity="1" transform="matrix(1 0 0 -1 2931.5 1542.02)"/><path d="M652.5 1182.5C638.969 1182.5 628 1180.67 628 1178.42L628 861.583C628 859.328 617.031 857.5 603.5 857.5 617.031 857.5 628 855.672 628 853.417L628 536.583C628 534.328 638.97 532.5 652.501 532.5" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none" fill-rule="evenodd"/><path d="M651.5 1867.5C637.969 1867.5 627 1865.67 627 1863.42L627 1547.08C627 1544.83 616.031 1543 602.5 1543 616.031 1543 627 1541.17 627 1538.92L627 1222.58C627 1220.33 637.97 1218.5 651.501 1218.5" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none" fill-rule="evenodd"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 331.324 679)">tensor </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 286.339 756)">reduction </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 316.704 833)">group 1</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 402.779 910)">=</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 308.522 987)">pipeline </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 316.062 1064)">parallel </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 316.704 1141)">group 1</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 314.139 1313)">tensor </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 269.154 1390)">reduction </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 299.519 1467)">group 2</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 385.594 1544)">=</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 291.337 1621)">pipeline </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 298.877 1698)">parallel </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 299.519 1775)">group 2</text></g></g></svg>
\ No newline at end of file
docs/debug/img/reduction1.svg 0 → 100644
View file @ 2b05e121
This diff is collapsed.
docs/debug/img/reduction2.svg 0 → 100644
View file @ 2b05e121
<svg width="3250" height="1773" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xml:space="preserve" overflow="hidden"><g transform="translate(-499 -449)"><g><rect x="594.5" y="829.5" width="497" height="577" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none"/><rect x="1258.5" y="829.5" width="496" height="577" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none"/><path d="M1430.5 1186.5 1582.5 1186.5 1582.5 1363.5 1430.5 1363.5 1430.5 1312.05C1450.75 1312.05 1467.17 1296.82 1467.17 1278.04 1467.17 1259.25 1450.75 1244.02 1430.5 1244.02Z" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#D9F2D0" fill-rule="evenodd" fill-opacity="1"/><path d="M1430.5 895.5 1582.77 895.5 1582.77 949.68C1603.05 949.68 1619.5 964.821 1619.5 983.5 1619.5 1002.18 1603.05 1017.32 1582.77 1017.32L1582.77 1071.5 1430.5 1071.5 1430.5 1017 1443.06 1014.66C1456.24 1009.53 1465.49 997.509 1465.49 983.5 1465.49 969.491 1456.24 957.472 1443.06 952.338L1430.5 950.003Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M748.5 895.5 900.768 895.5 900.768 949.68C921.055 949.68 937.5 964.821 937.5 983.5 937.5 1002.18 921.055 1017.32 900.768 1017.32L900.768 1071.5 748.5 1071.5Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M748.5 1186.5 900.768 1186.5 900.768 1240.99C921.055 1240.99 937.5 1256.22 937.5 1275 937.5 1293.78 921.055 1309.01 900.768 1309.01L900.768 1363.5 748.5 1363.5Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#D9F2D0" fill-rule="evenodd" fill-opacity="1"/><rect x="1876.5" y="829.5" width="496" height="577" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none"/><rect x="2539.5" y="829.5" width="497" height="577" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none"/><path d="M2711.5 1186.5 2864.5 1186.5 2864.5 1363.5 2711.5 1363.5 2711.5 1312.05C2731.88 1312.05 2748.41 1296.82 2748.41 1278.04 2748.41 1259.25 2731.88 1244.02 2711.5 1244.02Z" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#D9F2D0" fill-rule="evenodd" fill-opacity="1"/><path d="M2030.5 1186.5 2182.77 1186.5 2182.77 1240.99C2203.05 1240.99 2219.5 1256.22 2219.5 1275 2219.5 1293.78 2203.05 1309.01 2182.77 1309.01L2182.77 1363.5 2030.5 1363.5Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#D9F2D0" fill-rule="evenodd" fill-opacity="1"/><path d="M2711.5 895.5 2864.5 895.5 2864.5 1071.5 2711.5 1071.5 2711.5 1020.34C2731.88 1020.34 2748.41 1005.2 2748.41 986.519 2748.41 967.841 2731.88 952.699 2711.5 952.699Z" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M2030.5 895.5 2182.77 895.5 2182.77 949.68C2203.05 949.68 2219.5 964.821 2219.5 983.5 2219.5 1002.18 2203.05 1017.32 2182.77 1017.32L2182.77 1071.5 2030.5 1071.5 2030.5 1017 2043.06 1014.66C2056.24 1009.53 2065.49 997.509 2065.49 983.5 2065.49 969.491 2056.24 957.472 2043.06 952.338L2030.5 950.003Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M784.5 1617.5 878.762 1617.5 878.762 1655.36C891.32 1655.36 901.5 1665.95 901.5 1679 901.5 1692.05 891.32 1702.64 878.762 1702.64L878.762 1740.5 784.5 1740.5Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M818.5 1788.5 945.5 1788.5 945.5 1911.5 818.5 1911.5 818.5 1875.75C835.42 1875.75 849.136 1865.16 849.136 1852.11 849.136 1839.06 835.42 1828.47 818.5 1828.47Z" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#D9F2D0" fill-rule="evenodd" fill-opacity="1"/><path d="M740.5 1788.5 835.567 1788.5 835.567 1826.36C848.233 1826.36 858.5 1836.95 858.5 1850 858.5 1863.05 848.233 1873.64 835.567 1873.64L835.567 1911.5 740.5 1911.5Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#D9F2D0" fill-rule="evenodd" fill-opacity="1"/><path d="M1445.5 1614.5 1543.79 1614.5 1543.79 1652.36C1556.88 1652.36 1567.5 1662.95 1567.5 1676 1567.5 1689.05 1556.88 1699.64 1543.79 1699.64L1543.79 1737.5 1445.5 1737.5 1445.5 1699.41 1453.61 1697.78C1462.12 1694.19 1468.09 1685.79 1468.09 1676 1468.09 1666.21 1462.12 1657.81 1453.61 1654.22L1445.5 1652.59Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M1473.5 1789.5 1599.5 1789.5 1599.5 1911.5 1473.5 1911.5 1473.5 1876.04C1490.29 1876.04 1503.89 1865.54 1503.89 1852.59 1503.89 1839.65 1490.29 1829.15 1473.5 1829.15Z" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#D9F2D0" fill-rule="evenodd" fill-opacity="1"/><path d="M1395.5 1789.5 1489.76 1789.5 1489.76 1827.06C1502.32 1827.06 1512.5 1837.55 1512.5 1850.5 1512.5 1863.45 1502.32 1873.94 1489.76 1873.94L1489.76 1911.5 1395.5 1911.5Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#D9F2D0" fill-rule="evenodd" fill-opacity="1"/><path d="M2063.5 1611.5 2161.79 1611.5 2161.79 1649.36C2174.88 1649.36 2185.5 1659.95 2185.5 1673 2185.5 1686.05 2174.88 1696.64 2161.79 1696.64L2161.79 1734.5 2063.5 1734.5 2063.5 1696.41 2071.61 1694.78C2080.12 1691.19 2086.09 1682.79 2086.09 1673 2086.09 1663.21 2080.12 1654.81 2071.61 1651.22L2063.5 1649.59Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M2088.5 1788.5 2214.5 1788.5 2214.5 1911.5 2088.5 1911.5 2088.5 1875.75C2105.29 1875.75 2118.89 1865.16 2118.89 1852.11 2118.89 1839.06 2105.29 1828.47 2088.5 1828.47Z" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#D9F2D0" fill-rule="evenodd" fill-opacity="1"/><path d="M2010.5 1788.5 2104.76 1788.5 2104.76 1826.36C2117.32 1826.36 2127.5 1836.95 2127.5 1850 2127.5 1863.05 2117.32 1873.64 2104.76 1873.64L2104.76 1911.5 2010.5 1911.5Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#D9F2D0" fill-rule="evenodd" fill-opacity="1"/><path d="M2733.5 1617.5 2842.5 1617.5 2842.5 1740.5 2733.5 1740.5 2733.5 1704.7 2743.5 1702.89C2752.96 1699.3 2759.6 1690.9 2759.6 1681.11L2759.49 1680.63 2759.86 1679C2759.86 1665.95 2748.06 1655.36 2733.5 1655.36Z" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M2767.5 1788.5 2893.5 1788.5 2893.5 1911.5 2767.5 1911.5 2767.5 1875.75C2784.29 1875.75 2797.89 1865.16 2797.89 1852.11 2797.89 1839.06 2784.29 1828.47 2767.5 1828.47Z" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#D9F2D0" fill-rule="evenodd" fill-opacity="1"/><path d="M2689.5 1788.5 2783.76 1788.5 2783.76 1826.36C2796.32 1826.36 2806.5 1836.95 2806.5 1850 2806.5 1863.05 2796.32 1873.64 2783.76 1873.64L2783.76 1911.5 2689.5 1911.5Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#D9F2D0" fill-rule="evenodd" fill-opacity="1"/><path d="M577.5 655.5C577.5 630.923 580.82 611 584.917 611L827.083 611C831.179 611 834.5 591.077 834.5 566.5 834.5 591.077 837.82 611 841.917 611L1084.08 611C1088.18 611 1091.5 630.923 1091.5 655.5" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none" fill-rule="evenodd"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1031.56 731)">TP group 1</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 3175.14 1003)">activation/gradient </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 3330.42 1080)">tensors</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 3215.24 1294)">weight tensors</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="55" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 584.658 519)">Tensor reduction group</text><path d="M577.5 838.5C577.5 813.923 580.82 793.999 584.917 793.999L1175.08 793.999C1179.18 793.999 1182.5 774.076 1182.5 749.499 1182.5 774.076 1185.82 793.999 1189.92 793.999L1780.08 793.999C1784.18 793.999 1787.5 813.923 1787.5 838.5" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none" fill-rule="evenodd"/><path d="M1848.5 836.5C1848.5 811.923 1851.82 791.999 1855.92 791.999L2446.08 791.999C2450.18 791.999 2453.5 772.076 2453.5 747.499 2453.5 772.076 2456.82 791.999 2460.92 791.999L3051.08 791.999C3055.18 791.999 3058.5 811.923 3058.5 836.5" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none" fill-rule="evenodd"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2285.23 713)">TP group 2</text><path d="M841.937 1945.5 841.938 2033.56 835.063 2033.56 835.062 1945.5ZM852.25 2028.97 838.5 2056.47 824.75 2028.97Z" fill="#000000" fill-rule="nonzero" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 771.787 2164)">Stats</text><path d="M1491.94 1945.5 1491.94 2033.56 1485.06 2033.56 1485.06 1945.5ZM1502.25 2028.97 1488.5 2056.47 1474.75 2028.97Z" fill="#000000" fill-rule="nonzero" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1421.78 2164)">Stats</text><path d="M2106.94 1945.5 2106.94 2033.56 2100.06 2033.56 2100.06 1945.5ZM2117.25 2028.97 2103.5 2056.47 2089.75 2028.97Z" fill="#000000" fill-rule="nonzero" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2036.58 2164)">Stats</text><path d="M2786.94 1945.5 2786.94 2033.56 2780.06 2033.56 2780.06 1945.5ZM2797.25 2028.97 2783.5 2056.47 2769.75 2028.97Z" fill="#000000" fill-rule="nonzero" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2716.75 2164)">Stats</text><path d="M1237.5 655.5C1237.5 630.923 1240.82 611 1244.92 611L1487.08 611C1491.18 611 1494.5 591.077 1494.5 566.5 1494.5 591.077 1497.82 611 1501.92 611L1744.08 611C1748.18 611 1751.5 630.923 1751.5 655.5" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none" fill-rule="evenodd"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="55" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1245.05 519)">Tensor reduction group</text><path d="M1846.5 656.5C1846.5 631.923 1849.82 612 1853.92 612L2096.08 612C2100.18 612 2103.5 592.077 2103.5 567.5 2103.5 592.077 2106.82 612 2110.92 612L2353.08 612C2357.18 612 2360.5 631.923 2360.5 656.5" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none" fill-rule="evenodd"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="55" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1853.18 520)">Tensor reduction group</text><path d="M2531.5 656.5C2531.5 631.647 2534.86 611.5 2539 611.5L2781 611.5C2785.14 611.5 2788.5 591.353 2788.5 566.5 2788.5 591.353 2791.86 611.5 2796 611.5L3038 611.5C3042.14 611.5 3045.5 631.647 3045.5 656.5" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none" fill-rule="evenodd"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="55" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2538.46 520)">Tensor reduction group</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 743.462 1529)">Node </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 907.316 1529)">1</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1393.45 1529)">Node </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1557.31 1529)">2</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2008.25 1529)">Node </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2172.11 1529)">3</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2688.42 1530)">Node </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2852.28 1530)">4</text></g></g></svg>
\ No newline at end of file
docs/debug/img/reduction3.svg 0 → 100644
View file @ 2b05e121
<svg width="3177" height="1801" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xml:space="preserve" overflow="hidden"><g transform="translate(-572 -421)"><g><rect x="594.5" y="829.5" width="497" height="577" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none"/><rect x="1258.5" y="829.5" width="496" height="577" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none"/><path d="M1430.5 1186.5 1582.5 1186.5 1582.5 1363.5 1430.5 1363.5 1430.5 1312.05C1450.75 1312.05 1467.17 1296.82 1467.17 1278.04 1467.17 1259.25 1450.75 1244.02 1430.5 1244.02Z" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#D9F2D0" fill-rule="evenodd" fill-opacity="1"/><path d="M1430.5 895.5 1582.77 895.5 1582.77 949.68C1603.05 949.68 1619.5 964.821 1619.5 983.5 1619.5 1002.18 1603.05 1017.32 1582.77 1017.32L1582.77 1071.5 1430.5 1071.5 1430.5 1017 1443.06 1014.66C1456.24 1009.53 1465.49 997.509 1465.49 983.5 1465.49 969.491 1456.24 957.472 1443.06 952.338L1430.5 950.003Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M748.5 895.5 900.768 895.5 900.768 949.68C921.055 949.68 937.5 964.821 937.5 983.5 937.5 1002.18 921.055 1017.32 900.768 1017.32L900.768 1071.5 748.5 1071.5Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M748.5 1186.5 900.768 1186.5 900.768 1240.99C921.055 1240.99 937.5 1256.22 937.5 1275 937.5 1293.78 921.055 1309.01 900.768 1309.01L900.768 1363.5 748.5 1363.5Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#D9F2D0" fill-rule="evenodd" fill-opacity="1"/><rect x="1876.5" y="829.5" width="496" height="577" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none"/><rect x="2539.5" y="829.5" width="497" height="577" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none"/><path d="M2711.5 1186.5 2864.5 1186.5 2864.5 1363.5 2711.5 1363.5 2711.5 1312.05C2731.88 1312.05 2748.41 1296.82 2748.41 1278.04 2748.41 1259.25 2731.88 1244.02 2711.5 1244.02Z" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#D9F2D0" fill-rule="evenodd" fill-opacity="1"/><path d="M2030.5 1186.5 2182.77 1186.5 2182.77 1240.99C2203.05 1240.99 2219.5 1256.22 2219.5 1275 2219.5 1293.78 2203.05 1309.01 2182.77 1309.01L2182.77 1363.5 2030.5 1363.5Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#D9F2D0" fill-rule="evenodd" fill-opacity="1"/><path d="M2711.5 895.5 2864.5 895.5 2864.5 1071.5 2711.5 1071.5 2711.5 1020.34C2731.88 1020.34 2748.41 1005.2 2748.41 986.519 2748.41 967.841 2731.88 952.699 2711.5 952.699Z" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M2030.5 895.5 2182.77 895.5 2182.77 949.68C2203.05 949.68 2219.5 964.821 2219.5 983.5 2219.5 1002.18 2203.05 1017.32 2182.77 1017.32L2182.77 1071.5 2030.5 1071.5 2030.5 1017 2043.06 1014.66C2056.24 1009.53 2065.49 997.509 2065.49 983.5 2065.49 969.491 2056.24 957.472 2043.06 952.338L2030.5 950.003Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M743.5 1611.5 841.79 1611.5 841.79 1649.36C854.885 1649.36 865.5 1659.95 865.5 1673 865.5 1686.05 854.885 1696.64 841.79 1696.64L841.79 1734.5 743.5 1734.5 743.5 1696.41 751.607 1694.78C760.117 1691.19 766.088 1682.79 766.088 1673 766.088 1663.21 760.117 1654.81 751.607 1651.22L743.5 1649.59Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M653.5 1611.5 748.567 1611.5 748.567 1649.36C761.233 1649.36 771.5 1659.95 771.5 1673 771.5 1686.05 761.233 1696.64 748.567 1696.64L748.567 1734.5 653.5 1734.5Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M897.5 1611.5 1032.5 1611.5 1032.5 1734.5 897.5 1734.5 897.5 1698.7 909.883 1696.89C921.6 1693.3 929.821 1684.9 929.821 1675.11L929.688 1674.63 930.144 1673C930.144 1659.95 915.529 1649.36 897.5 1649.36Z" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M838.5 1611.5 936.79 1611.5 936.79 1649.36C949.885 1649.36 960.5 1659.95 960.5 1673 960.5 1686.05 949.885 1696.64 936.79 1696.64L936.79 1734.5 838.5 1734.5 838.5 1696.41 846.607 1694.78C855.117 1691.19 861.088 1682.79 861.088 1673 861.088 1663.21 855.117 1654.81 846.607 1651.22L838.5 1649.59Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M794.5 1789.5 888.762 1789.5 888.762 1827.36C901.32 1827.36 911.5 1837.95 911.5 1851 911.5 1864.05 901.32 1874.64 888.762 1874.64L888.762 1912.5 794.5 1912.5Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#D9F2D0" fill-rule="evenodd" fill-opacity="1"/><path d="M1398.5 1612.5 1496.79 1612.5 1496.79 1650.36C1509.88 1650.36 1520.5 1660.95 1520.5 1674 1520.5 1687.05 1509.88 1697.64 1496.79 1697.64L1496.79 1735.5 1398.5 1735.5 1398.5 1697.41 1406.61 1695.78C1415.12 1692.19 1421.09 1683.79 1421.09 1674 1421.09 1664.21 1415.12 1655.81 1406.61 1652.22L1398.5 1650.59Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M1308.5 1612.5 1402.76 1612.5 1402.76 1650.36C1415.32 1650.36 1425.5 1660.95 1425.5 1674 1425.5 1687.05 1415.32 1697.64 1402.76 1697.64L1402.76 1735.5 1308.5 1735.5Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M1552.5 1612.5 1686.5 1612.5 1686.5 1735.5 1552.5 1735.5 1552.5 1699.7 1564.79 1697.89C1576.42 1694.3 1584.58 1685.9 1584.58 1676.11L1584.45 1675.63 1584.9 1674C1584.9 1660.95 1570.4 1650.36 1552.5 1650.36Z" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M1493.5 1612.5 1590.98 1612.5 1590.98 1650.36C1603.97 1650.36 1614.5 1660.95 1614.5 1674 1614.5 1687.05 1603.97 1697.64 1590.98 1697.64L1590.98 1735.5 1493.5 1735.5 1493.5 1697.41 1501.54 1695.78C1509.98 1692.19 1515.9 1683.79 1515.9 1674 1515.9 1664.21 1509.98 1655.81 1501.54 1652.22L1493.5 1650.59Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M1433.5 1788.5 1550.5 1788.5 1550.5 1911.5 1433.5 1911.5 1433.5 1875.75C1449.09 1875.75 1461.72 1865.16 1461.72 1852.11 1461.72 1839.06 1449.09 1828.47 1433.5 1828.47Z" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#D9F2D0" fill-rule="evenodd" fill-opacity="1"/><path d="M2013.5 1611.5 2111.79 1611.5 2111.79 1649.36C2124.88 1649.36 2135.5 1659.95 2135.5 1673 2135.5 1686.05 2124.88 1696.64 2111.79 1696.64L2111.79 1734.5 2013.5 1734.5 2013.5 1696.41 2021.61 1694.78C2030.12 1691.19 2036.09 1682.79 2036.09 1673 2036.09 1663.21 2030.12 1654.81 2021.61 1651.22L2013.5 1649.59Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M1923.5 1611.5 2017.76 1611.5 2017.76 1649.36C2030.32 1649.36 2040.5 1659.95 2040.5 1673 2040.5 1686.05 2030.32 1696.64 2017.76 1696.64L2017.76 1734.5 1923.5 1734.5Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M2167.5 1611.5 2301.5 1611.5 2301.5 1734.5 2167.5 1734.5 2167.5 1698.7 2179.79 1696.89C2191.42 1693.3 2199.58 1684.9 2199.58 1675.11L2199.45 1674.63 2199.9 1673C2199.9 1659.95 2185.39 1649.36 2167.5 1649.36Z" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M2107.5 1611.5 2205.79 1611.5 2205.79 1649.36C2218.88 1649.36 2229.5 1659.95 2229.5 1673 2229.5 1686.05 2218.88 1696.64 2205.79 1696.64L2205.79 1734.5 2107.5 1734.5 2107.5 1696.41 2115.61 1694.78C2124.12 1691.19 2130.09 1682.79 2130.09 1673 2130.09 1663.21 2124.12 1654.81 2115.61 1651.22L2107.5 1649.59Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M2063.5 1789.5 2157.76 1789.5 2157.76 1827.36C2170.32 1827.36 2180.5 1837.95 2180.5 1851 2180.5 1864.05 2170.32 1874.64 2157.76 1874.64L2157.76 1912.5 2063.5 1912.5Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#D9F2D0" fill-rule="evenodd" fill-opacity="1"/><path d="M2692.5 1611.5 2790.79 1611.5 2790.79 1649.36C2803.88 1649.36 2814.5 1659.95 2814.5 1673 2814.5 1686.05 2803.88 1696.64 2790.79 1696.64L2790.79 1734.5 2692.5 1734.5 2692.5 1696.41 2700.61 1694.78C2709.12 1691.19 2715.09 1682.79 2715.09 1673 2715.09 1663.21 2709.12 1654.81 2700.61 1651.22L2692.5 1649.59Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M2602.5 1611.5 2696.76 1611.5 2696.76 1649.36C2709.32 1649.36 2719.5 1659.95 2719.5 1673 2719.5 1686.05 2709.32 1696.64 2696.76 1696.64L2696.76 1734.5 2602.5 1734.5Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M2846.5 1611.5 2980.5 1611.5 2980.5 1734.5 2846.5 1734.5 2846.5 1698.7 2858.79 1696.89C2870.42 1693.3 2878.58 1684.9 2878.58 1675.11L2878.45 1674.63 2878.9 1673C2878.9 1659.95 2864.4 1649.36 2846.5 1649.36Z" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M2787.5 1611.5 2884.98 1611.5 2884.98 1649.36C2897.97 1649.36 2908.5 1659.95 2908.5 1673 2908.5 1686.05 2897.97 1696.64 2884.98 1696.64L2884.98 1734.5 2787.5 1734.5 2787.5 1696.41 2795.54 1694.78C2803.98 1691.19 2809.9 1682.79 2809.9 1673 2809.9 1663.21 2803.98 1654.81 2795.54 1651.22L2787.5 1649.59Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M2719.5 1789.5 2837.5 1789.5 2837.5 1912.5 2719.5 1912.5 2719.5 1876.75C2735.22 1876.75 2747.96 1866.16 2747.96 1853.11 2747.96 1840.06 2735.22 1829.47 2719.5 1829.47Z" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#D9F2D0" fill-rule="evenodd" fill-opacity="1"/><path d="M577.5 655.5C577.5 630.923 580.82 610.999 584.915 610.999L1810.58 610.999C1814.68 610.999 1818 591.075 1818 566.497 1818 591.075 1821.32 610.999 1825.42 610.999L3051.08 610.999C3055.18 610.999 3058.5 630.923 3058.5 655.5" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none" fill-rule="evenodd"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1036.2 713)">TP group 1</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 3175.14 1003)">activation/gradient </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 3330.42 1080)">tensors</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 3215.24 1294)">weight tensors</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1483.27 505)">Tensor reduction group</text><path d="M577.5 838.5C577.5 813.923 580.82 793.999 584.917 793.999L1175.08 793.999C1179.18 793.999 1182.5 774.076 1182.5 749.499 1182.5 774.076 1185.82 793.999 1189.92 793.999L1780.08 793.999C1784.18 793.999 1787.5 813.923 1787.5 838.5" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none" fill-rule="evenodd"/><path d="M1848.5 836.5C1848.5 811.923 1851.82 791.999 1855.92 791.999L2446.08 791.999C2450.18 791.999 2453.5 772.076 2453.5 747.499 2453.5 772.076 2456.82 791.999 2460.92 791.999L3051.08 791.999C3055.18 791.999 3058.5 811.923 3058.5 836.5" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none" fill-rule="evenodd"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2285.23 713)">TP group 2</text><path d="M841.937 1945.5 841.938 2033.56 835.063 2033.56 835.062 1945.5ZM852.25 2028.97 838.5 2056.47 824.75 2028.97Z" fill="#000000" fill-rule="nonzero" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 771.787 2164)">Stats</text><path d="M1491.94 1945.5 1491.94 2033.56 1485.06 2033.56 1485.06 1945.5ZM1502.25 2028.97 1488.5 2056.47 1474.75 2028.97Z" fill="#000000" fill-rule="nonzero" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1421.78 2164)">Stats</text><path d="M2106.94 1945.5 2106.94 2033.56 2100.06 2033.56 2100.06 1945.5ZM2117.25 2028.97 2103.5 2056.47 2089.75 2028.97Z" fill="#000000" fill-rule="nonzero" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2036.58 2164)">Stats</text><path d="M2786.94 1945.5 2786.94 2033.56 2780.06 2033.56 2780.06 1945.5ZM2797.25 2028.97 2783.5 2056.47 2769.75 2028.97Z" fill="#000000" fill-rule="nonzero" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2716.75 2164)">Stats</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 743.462 1507)">Node </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 907.316 1507)">1</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1393.45 1507)">Node </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1557.31 1507)">2</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2008.25 1507)">Node </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2172.11 1507)">3</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2688.42 1508)">Node </text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2852.28 1508)">4</text></g></g></svg>
\ No newline at end of file
docs/debug/img/scaling_factors.svg 0 → 100644
View file @ 2b05e121
<svg width="3350" height="1050" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xml:space="preserve" overflow="hidden"><g transform="translate(-252 -728)"><g><rect x="1012.5" y="1003.5" width="497" height="577" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none"/><rect x="1622.5" y="1003.5" width="497" height="577" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none"/><rect x="277.5" y="1003.5" width="497" height="577" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none"/><path d="M507.5 1222.5 633.5 1222.5 633.5 1345.5 507.5 1345.5 507.5 1309.75C524.286 1309.75 537.895 1299.16 537.895 1286.11 537.895 1273.06 524.286 1262.47 507.5 1262.47Z" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M429.5 1222.5 523.762 1222.5 523.762 1260.36C536.32 1260.36 546.5 1270.95 546.5 1284 546.5 1297.05 536.32 1307.64 523.762 1307.64L523.762 1345.5 429.5 1345.5Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M1202.5 1237.5 1296.76 1237.5 1296.76 1275.36C1309.32 1275.36 1319.5 1285.95 1319.5 1299 1319.5 1312.05 1309.32 1322.64 1296.76 1322.64L1296.76 1360.5 1202.5 1360.5Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M1807.5 1233.5 1934.5 1233.5 1934.5 1356.5 1807.5 1356.5 1807.5 1320.75C1824.42 1320.75 1838.14 1310.16 1838.14 1297.11 1838.14 1284.06 1824.42 1273.47 1807.5 1273.47Z" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M532.937 1360.5 532.938 1460.54 526.063 1460.54 526.062 1360.5ZM543.25 1455.95 529.5 1483.45 515.75 1455.95Z" fill="#156082" fill-rule="nonzero" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="50" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 322.919 1547)">One Scaling Factor</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="50" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1037.68 1547)">Scaling Factor No. 1</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="50" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1643.02 1548)">Scaling Factor No. 2</text><path d="M1870.94 1371.5 1870.94 1464.58 1864.06 1464.58 1864.06 1371.5ZM1881.25 1460 1867.5 1487.5 1853.75 1460Z" fill="#156082" fill-rule="nonzero" fill-opacity="1"/><path d="M1249.94 1378.5 1249.94 1471.58 1243.06 1471.58 1243.06 1378.5ZM1260.25 1467 1246.5 1494.5 1232.75 1467Z" fill="#156082" fill-rule="nonzero" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1170.72 975)">Node 1</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1757.09 975)">Node 2</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 453.544 975)">Node</text><path d="M2282.5 730.5 2282.5 1770.26" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none" fill-rule="evenodd"/><path d="M899.5 730.5 899.5 1770.26" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none" fill-rule="evenodd"/><rect x="2371.5" y="1003.5" width="497" height="577" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none"/><rect x="2981.5" y="1003.5" width="497" height="577" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="none"/><path d="M2561.5 1237.5 2655.76 1237.5 2655.76 1275.36C2668.32 1275.36 2678.5 1285.95 2678.5 1299 2678.5 1312.05 2668.32 1322.64 2655.76 1322.64L2655.76 1360.5 2561.5 1360.5Z" stroke="#000000" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><path d="M3166.5 1233.5 3293.5 1233.5 3293.5 1356.5 3166.5 1356.5 3166.5 1320.75C3183.42 1320.75 3197.14 1310.16 3197.14 1297.11 3197.14 1284.06 3183.42 1273.47 3166.5 1273.47Z" stroke="#042433" stroke-width="6.875" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="8" stroke-opacity="1" fill="#FBE3D6" fill-rule="evenodd" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="50" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2426.08 1547)">One Scaling Factor</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="50" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 3023.31 1548)">One Scaling Factor</text><path d="M3228.94 1371.5 3228.94 1464.58 3222.06 1464.58 3222.06 1371.5ZM3239.25 1460 3225.5 1487.5 3211.75 1460Z" fill="#156082" fill-rule="nonzero" fill-opacity="1"/><path d="M2607.94 1378.5 2607.94 1471.58 2601.06 1471.58 2601.06 1378.5ZM2618.25 1467 2604.5 1494.5 2590.75 1467Z" fill="#156082" fill-rule="nonzero" fill-opacity="1"/><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2529.3 975)">Node 1</text><text fill="#000000" fill-opacity="1" font-family="Aptos,Aptos_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="64" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 3115.67 975)">Node 2</text><text fill="#404040" fill-opacity="1" font-family="NVIDIA Sans,NVIDIA Sans_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="700" font-stretch="normal" font-size="73" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 959.215 839)">PerTensorScaling</text><text fill="#404040" fill-opacity="1" font-family="NVIDIA Sans,NVIDIA Sans_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="73" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1585.12 839)">and</text><text fill="#404040" fill-opacity="1" font-family="NVIDIA Sans,NVIDIA Sans_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="700" font-stretch="normal" font-size="73" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 1732.35 839)">FakeQuant</text><text fill="#404040" fill-opacity="1" font-family="NVIDIA Sans,NVIDIA Sans_MSFontService,sans-serif" font-style="normal" font-variant="normal" font-weight="400" font-stretch="normal" font-size="73" text-anchor="start" direction="ltr" writing-mode="lr-tb" unicode-bidi="normal" text-decoration="none" transform="matrix(1 0 0 1 2610.45 839)">FP8 Delayed Scaling</text></g></g></svg>
\ No newline at end of file
docs/examples/te_llama/te_llama.py
View file @ 2b05e121
......@@ -19,7 +19,7 @@ from transformers.models.llama.modeling_llama import (
LlamaRMSNorm,
LlamaConfig,
)
from transformers.modeling_utils import _add_variant, load_state_dict, _load_state_dict_into_model
from transformers.modeling_utils import _add_variant, load_state_dict
from transformers.utils import WEIGHTS_INDEX_NAME
from transformers.utils.hub import get_checkpoint_shard_files
......@@ -148,8 +148,8 @@ class TELlamaForCausalLM:
state_dict = load_state_dict(shard_file)
# replace_params copies parameters relevant only to TransformerEngine
replace_params(state_dict, vanilla_model.state_dict(), config)
# _load_state_dict_into_model copies parameters other than those in TransformerEngine
_load_state_dict_into_model(vanilla_model, state_dict, start_prefix="")
# load_state_dict copies parameters other than those in TransformerEngine
vanilla_model.load_state_dict(state_dict, strict=False)
# Force mem release. Taken from huggingface code
del state_dict
......
docs/index.rst
View file @ 2b05e121
......@@ -52,4 +52,5 @@ Transformer Engine documentation
:caption: Advanced
api/c/index
debug
examples/attention/attention.ipynb
docs/installation.rst
View file @ 2b05e121
......@@ -112,7 +112,7 @@ To build the C++ extensions with debug symbols, e.g. with the `-g` flag:
.. code-block:: bash
pip3 install --no-build-isolation . --global-option=--debug
NVTE_BUILD_DEBUG=1 pip3 install --no-build-isolation .
.. include:: ../README.rst
:start-after: troubleshooting-begin-marker-do-not-remove
......
examples/jax/encoder/common.py
View file @ 2b05e121
......@@ -4,9 +4,12 @@
"""Shared functions for the encoder tests"""
from functools import lru_cache
import jax
import jax.numpy
import transformer_engine
from transformer_engine_jax import get_device_compute_capability
from transformer_engine.common import recipe
import numpy as np
@lru_cache
......@@ -30,6 +33,71 @@ def is_mxfp8_supported():
return gpu_arch >= 100
def assert_params_sufficiently_sharded(params, mesh, tolerance=0.01, print_info=False):
"""Checks whether most params are sharded across sharding axis.
(Adapted from https://github.com/AI-Hypercomputer/maxtext/blob/315e551e5942b24656a4250dcfca986fb4135b72/MaxText/maxtext_utils.py#L348)
This function determines whether the majority of parameters are distributed
across a specified sharding axes with an acceptable tolerance. It compares the
current distribution to a scenario where all parameters are fully sharded
across the axes on which the params are sharded e.g. 'tensor' axis.
Args:
params: params of the model state
mesh: mesh constructed from config
tolerance: float between 0.0 and 1.0 representing the allowed percentage of
non-sharded parameters.
"""
def get_product_num_devices_for_weight_sharding(weight_sharding_axes):
product_num_devices_for_weight_sharding = 1
for axis in weight_sharding_axes:
product_num_devices_for_weight_sharding *= mesh.shape.get(axis, 1)
return product_num_devices_for_weight_sharding
def assert_leaf_sharding(path, arr):
# Is the weight sharded? Get the axes on which it is sharded.
partition_spec = arr.sharding.spec
weight_sharding_axes = set(partition_spec) - set([None]) # None is not a sharding axis
# Total number of devices on the axes on which the weight is sharded.
product_num_devices_for_weight_sharding = get_product_num_devices_for_weight_sharding(
weight_sharding_axes
)
# Params present in one shard (on one device).
shard = arr.addressable_shards[0]
params_per_chip = np.prod(shard.data.shape)
# Total number of params (across all devicess).
total_params = jax.numpy.size(arr)
# Percentage of params that are unsharded.
unsharded_perc = (
(params_per_chip / (total_params / product_num_devices_for_weight_sharding) - 1) * 100
if params_per_chip < total_params
else 100
)
if print_info:
print(
f"{path}: {unsharded_perc:.2f}% unsharded, unsharded param shape={arr.shape},"
f" partition spec={partition_spec}"
)
# If the weight is sharded on any axis, then the percentage of
# unsharded params should be less than the tolerance.
assert (
product_num_devices_for_weight_sharding == 1 or unsharded_perc < tolerance
), f"{path}: {unsharded_perc:.2f}% unsharded"
jax.tree_util.tree_map_with_path(
lambda p, x: assert_leaf_sharding("/".join(str(x) for x in p), x), params
)
def get_fp8_recipe_from_name_string(name: str):
"""Query recipe from a given name string"""
match name:
......
examples/jax/encoder/run_test_multiprocessing_encoder.sh
View file @ 2b05e121
......@@ -6,13 +6,13 @@ NUM_GPUS=${NUM_GPUS:-$(nvidia-smi -L | wc -l)}
# Define the test cases to run
TEST_CASES=(
"test_te_bf16"
# "test_te_bf16"
"test_te_delayed_scaling_fp8"
"test_te_current_scaling_fp8"
"test_te_mxfp8"
"test_te_bf16_shardy"
# "test_te_current_scaling_fp8"
# "test_te_mxfp8"
# "test_te_bf16_shardy"
"test_te_delayed_scaling_fp8_shardy"
"test_te_current_scaling_fp8_shardy"
# "test_te_current_scaling_fp8_shardy"
)
echo
......@@ -38,12 +38,13 @@ for TEST_CASE in "${TEST_CASES[@]}"; do
# Wait for the process to finish
wait
tail -n +7 "${TEST_CASE}_gpu_0.log"
tail -n +7 "${TEST_CASE}_gpu_0.log"
# Check and print the log content accordingly
if grep -q "FAILED" "${TEST_CASE}_gpu_0.log"; then
HAS_FAILURE=1
echo "... $TEST_CASE FAILED"
tail -n +7 "${TEST_CASE}_gpu_0.log"
elif grep -q "SKIPPED" "${TEST_CASE}_gpu_0.log"; then
echo "... $TEST_CASE SKIPPED"
elif grep -q "PASSED" "${TEST_CASE}_gpu_0.log"; then
......
examples/jax/encoder/test_model_parallel_encoder.py
View file @ 2b05e121
......@@ -19,7 +19,11 @@ from flax.training import train_state
from jax.experimental import mesh_utils
from jax.sharding import PartitionSpec, NamedSharding
from common import is_bf16_supported, get_fp8_recipe_from_name_string
from common import (
is_bf16_supported,
get_fp8_recipe_from_name_string,
assert_params_sufficiently_sharded,
)
import transformer_engine.jax as te
import transformer_engine.jax.flax as te_flax
from transformer_engine.jax.quantize import is_fp8_available, ScalingMode
......@@ -223,38 +227,6 @@ def check_fp8(state, var_collect, inputs, masks, labels):
assert "f8_e5m2" in func_jaxpr or "f8_e4m3" in func_jaxpr
def get_params_sharding(sharding_rules, abs_var_collect, mesh):
"""Refer params to create params sharding"""
rules_dict = dict(sharding_rules)
def to_device_axis(logical_axis):
partitions = [rules_dict[key] for key in logical_axis]
return NamedSharding(mesh, PartitionSpec(*partitions))
params_axes = abs_var_collect.get(PARAMS_AXES_KEY, {})
params_axes_sharding = jax.tree_util.tree_map(
to_device_axis, nn_partitioning.get_axis_names(params_axes)
)
params_axes_sharding = flax.core.unfreeze(params_axes_sharding)
params_sharding = jax.tree_util.tree_map(
lambda x: NamedSharding(mesh, PartitionSpec(None)), abs_var_collect[PARAMS_KEY]
)
params_sharding = {**params_sharding, **params_axes_sharding}
return params_sharding
def get_state_sharding(state, params_sharding):
"""Refer params_sharding to create state sharding"""
def replace_params(x):
return params_sharding if isinstance(x, dict) else None
state_sharding = jax.tree_util.tree_map(
replace_params, state, is_leaf=lambda x: isinstance(x, dict)
)
return state_sharding
def train_and_evaluate(args):
"""Execute model training and evaluation loop."""
print(args)
......@@ -291,8 +263,9 @@ def train_and_evaluate(args):
device_mesh = mesh_utils.create_device_mesh((num_gpu_dp, num_gpu_tp))
with jax.sharding.Mesh(
devices=device_mesh, axis_names=(DEVICE_DP_AXIS, DEVICE_TP_AXIS)
) as mesh:
) as mesh, nn_partitioning.axis_rules(
((NAMED_BROADCAST_AXIS, None), (NAMED_TP_AXIS, DEVICE_TP_AXIS))
):
rng = jax.random.PRNGKey(args.seed)
rng, params_rng = jax.random.split(rng)
rng, dropout_rng = jax.random.split(rng)
......@@ -312,25 +285,54 @@ def train_and_evaluate(args):
masks = jnp.zeros(mask_shape, dtype=jnp.uint8)
abs_var_collect = jax.eval_shape(encoder.init, init_rngs, inputs, masks)
customized_rules = ((NAMED_BROADCAST_AXIS, None), (NAMED_TP_AXIS, DEVICE_TP_AXIS))
sharding_rules = te_flax.extend_logical_axis_rules(tuple()) + customized_rules
params_sharding = get_params_sharding(sharding_rules, abs_var_collect, mesh)
# Get the base axis rules and extend them with TE's rules.
axis_rules = nn_partitioning.get_axis_rules()
te_extended_axis_rules = te_flax.extend_logical_axis_rules(axis_rules)
print(f"Device mesh: {mesh}")
print(f"Axis rules: {te_extended_axis_rules}")
logical_partition_spec = nn.get_partition_spec(abs_var_collect)
# Note that `nn.logical_to_mesh_sharding` returns a dict with an extra
# "params" key that contains the sharding for the parameters.
params_sharding = nn.logical_to_mesh_sharding(
logical_partition_spec, mesh, te_extended_axis_rules
)
inputs_sharding = NamedSharding(mesh, PartitionSpec(DEVICE_DP_AXIS, None))
masks_sharding = NamedSharding(mesh, PartitionSpec(DEVICE_DP_AXIS, None, None, None))
in_shardings = (None, inputs_sharding, masks_sharding)
out_shardings = {
key: params_sharding if key is PARAMS_KEY else None for key in abs_var_collect
key: params_sharding[PARAMS_KEY] if key is PARAMS_KEY else None
for key in abs_var_collect
}
jit_encoder_init = jax.jit(encoder.init, in_shardings, out_shardings)
var_collect = jit_encoder_init(init_rngs, inputs, masks)
# Check if params are sufficiently sharded after initialization
assert_params_sufficiently_sharded(var_collect, mesh, print_info=False)
optimizer = optax.adamw(args.lr)
var_collect, params = flax.core.pop(var_collect, PARAMS_KEY)
state = train_state.TrainState.create(
apply_fn=encoder.apply, params=params, tx=optimizer
)
state_sharding = get_state_sharding(state, params_sharding)
abs_state = jax.eval_shape(
lambda: train_state.TrainState.create(
apply_fn=encoder.apply, params=params, tx=optimizer
)
)
logical_state_partition_spec = nn.get_partition_spec(abs_state)
state_sharding = nn.logical_to_mesh_sharding(
logical_state_partition_spec, mesh, te_extended_axis_rules
)
# Check if params are sufficiently sharded after jitting the state creation
assert_params_sufficiently_sharded(state.params, mesh, print_info=False)
# state_sharding = get_state_sharding(state, params_sharding)
labels_sharding = NamedSharding(mesh, PartitionSpec(DEVICE_DP_AXIS))
in_shardings = (
......
examples/jax/encoder/test_multiprocessing_encoder.py
View file @ 2b05e121
......@@ -609,7 +609,7 @@ class TestEncoder(unittest.TestCase):
def test_te_delayed_scaling_fp8(self):
"""Test Transformer Engine with DelayedScaling FP8"""
result = self.exec(True, "DelayedScaling")
assert result[0] < 0.505 and result[1] > 0.753
assert result[0] < 0.506 and result[1] > 0.753
@unittest.skipIf(
not is_fp8_supported(), "Device compute capability 9.0+ is required for CurrentScaling FP8"
......@@ -639,7 +639,7 @@ class TestEncoder(unittest.TestCase):
def test_te_delayed_scaling_fp8_shardy(self):
"""Test Transformer Engine with DelayedScaling FP8"""
result = self.exec(True, "DelayedScaling", enable_shardy=True)
assert result[0] < 0.505 and result[1] > 0.753
assert result[0] < 0.506 and result[1] > 0.753
# TODO(jreiffers): Add mxfp8 Shardy tests once supported in JAX.
......
pyproject.toml 0 → 100755
View file @ 2b05e121
# Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# See LICENSE for license information.
[build-system]
requires = ["setuptools>=61.0", "cmake>=3.21", "wheel", "pybind11[global]", "ninja", "pip", "torch>=2.1", "jax[cuda12]", "flax>=0.7.1"]
# Use legacy backend to import local packages in setup.py
build-backend = "setuptools.build_meta:__legacy__"
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment