add

tests/functional_tests/test_cases/gpt/gpt3_nightly_mcore_te_tp2_pp1_modelopt_distill_resume/model_config.yaml

@@ -62,4 +62,5 @@ MODEL_ARGS:
   --ckpt-format: torch_dist
   --data-cache-path: ${DATA_CACHE_PATH}
   --bf16: true
+  --attention-backend: unfused
 TEST_TYPE: ckpt-resume

tests/functional_tests/test_cases/multimodal-llava/multimodal_llava_mr_mcore_te_tp4_pp1_freeze_vit_freeze_lm_dgx_a100_1N8G/golden_values_dev.json

+{"forward-backward-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [7.96777, 0.62507, 0.62176, 0.62042, 0.62061, 0.62067, 0.62001, 0.61924, 0.61823, 0.6178]}, "forward-compute-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [3.04896, 0.30356, 0.30062, 0.29886, 0.29955, 0.29936, 0.29825, 0.29839, 0.2968, 0.29625]}, "backward-compute-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [0.99454, 0.28657, 0.28691, 0.28667, 0.28654, 0.28672, 0.28654, 0.2861, 0.28657, 0.28683]}, "batch-generator-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [0.3938, 0.01749, 0.01695, 0.01841, 0.01751, 0.01736, 0.01792, 0.01739, 0.01667, 0.01628]}, "forward-recv-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [3.32161, 0.03012, 0.02986, 0.02994, 0.02968, 0.02964, 0.03016, 0.02977, 0.02991, 0.02985]}, "forward-send-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [0.53192, 0.00018, 0.00018, 0.00018, 0.00019, 0.0002, 0.00019, 0.00019, 0.00019, 0.00018]}, "backward-recv-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [0.07283, 0.07198, 0.07135, 0.07044, 0.07023, 0.07085, 0.07065, 0.07057, 0.0704, 0.07021]}, "backward-send-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [0.00023, 0.00029, 0.0002, 0.00027, 0.00027, 0.00032, 0.00032, 0.00028, 0.00027, 0.00021]}, "forward-send-backward-recv-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [3.46399, 0.30175, 0.30094, 0.29597, 0.29703, 0.29641, 0.2959, 0.29432, 0.29344, 0.29317]}, "backward-send-forward-recv-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [0.82172, 0.00243, 0.00247, 0.00234, 0.00236, 0.00228, 0.0023, 0.00235, 0.00232, 0.00233]}, "layernorm-grads-all-reduce-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [4e-05, 3e-05, 3e-05, 3e-05, 3e-05, 2e-05, 3e-05, 3e-05, 3e-05, 3e-05]}, "embedding-grads-all-reduce-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [7e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05]}, "all-grads-sync-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [0.16382, 0.00025, 0.00025, 0.00025, 0.00024, 0.00024, 0.00024, 0.00024, 0.00023, 0.00026]}, "optimizer-copy-to-main-grad-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [7e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05]}, "optimizer-clip-main-grad-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [0.83319, 0.00053, 0.00052, 0.00044, 0.00052, 0.00043, 0.00043, 0.00043, 0.00043, 0.00043]}, "optimizer-count-zeros-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [0.00895, 0.00069, 0.00069, 0.00068, 0.00069, 0.00069, 0.00068, 0.00068, 0.00068, 0.00069]}, "optimizer-inner-step-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [0.00119, 0.00025, 0.00024, 0.00023, 0.00023, 0.00025, 0.00024, 0.00024, 0.00024, 0.00025]}, "optimizer-copy-main-to-model-params-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [0.00014, 9e-05, 9e-05, 8e-05, 8e-05, 9e-05, 9e-05, 8e-05, 9e-05, 9e-05]}, "optimizer-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [0.84455, 0.00225, 0.00226, 0.00214, 0.00221, 0.00216, 0.00214, 0.00213, 0.00214, 0.00214]}, "learning-rate": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, "learning-rate vs samples": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, "batch-size": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [32.0, 32.0, 32.0, 32.0, 32.0, 32.0, 32.0, 32.0, 32.0, 32.0]}, "batch-size vs samples": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [32.0, 32.0, 32.0, 32.0, 32.0, 32.0, 32.0, 32.0, 32.0, 32.0]}, "lm loss": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [9.19947, 9.20335, 9.20248, 9.19723, 9.19172, 9.18973, 9.18517, 9.17532, 9.17374, 9.1609]}, "lm loss vs samples": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [9.19947, 9.20335, 9.20248, 9.19723, 9.19172, 9.18973, 9.18517, 9.17532, 9.17374, 9.1609]}, "loss-scale": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, "loss-scale vs samples": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, "grad-norm": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [0.93277, 1.00171, 1.00056, 0.944, 1.16867, 0.98576, 0.91686, 0.9042, 0.83078, 0.88219]}, "grad-norm vs samples": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [0.93277, 1.00171, 1.00056, 0.944, 1.16867, 0.98576, 0.91686, 0.9042, 0.83078, 0.88219]}, "num-zeros": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [106.0, 114.0, 108.0, 110.0, 81.0, 105.0, 85.0, 109.0, 146.0, 122.0]}, "num-zeros vs samples": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [106.0, 114.0, 108.0, 110.0, 81.0, 105.0, 85.0, 109.0, 146.0, 122.0]}, "params-norm": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [35.67851, 35.67851, 35.67851, 35.67851, 35.67851, 35.6785, 35.67851, 35.6785, 35.67848, 35.67848]}, "params-norm vs samples": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [35.67851, 35.67851, 35.67851, 35.67851, 35.67851, 35.6785, 35.67851, 35.6785, 35.67848, 35.67848]}, "iteration-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [8.83079, 0.64044, 0.63692, 0.63516, 0.63554, 0.63541, 0.63471, 0.63399, 0.63285, 0.63245]}, "lm loss validation": {"start_step": 0, "end_step": 2, "step_interval": 5, "values": [9.1542]}, "lm loss validation vs samples": {"start_step": 0, "end_step": 1, "step_interval": 5, "values": [9.1542]}, "lm loss validation ppl": {"start_step": 0, "end_step": 1, "step_interval": 5, "values": [9454.09668]}, "lm loss validation ppl vs samples": {"start_step": 0, "end_step": 1, "step_interval": 5, "values": [9454.09668]}}
\ No newline at end of file

tests/functional_tests/test_cases/multimodal-llava/multimodal_llava_mr_mcore_te_tp4_pp1_freeze_vit_freeze_lm_dgx_a100_1N8G/golden_values_lts.json

+{"forward-backward-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [12.7291, 0.62672, 0.60589, 0.60528, 0.60867, 0.60545, 0.60403, 0.61268, 0.61851, 0.60357]}, "forward-compute-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [5.56178, 0.30066, 0.28459, 0.28176, 0.28541, 0.27947, 0.28138, 0.28895, 0.29453, 0.28039]}, "backward-compute-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [1.12115, 0.28858, 0.28597, 0.28809, 0.28772, 0.28811, 0.28721, 0.28849, 0.28849, 0.28829]}, "batch-generator-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [2.85702, 0.03903, 0.0338, 0.03035, 0.03224, 0.03016, 0.02978, 0.03435, 0.03368, 0.02954]}, "forward-recv-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [5.26228, 0.03127, 0.02963, 0.02987, 0.02952, 0.03226, 0.02962, 0.02934, 0.02956, 0.02928]}, "forward-send-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [2.5072, 0.00017, 0.00015, 0.00018, 0.00016, 0.00015, 0.00015, 0.00015, 0.00017, 0.00015]}, "backward-recv-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [0.07163, 0.07147, 0.0696, 0.06982, 0.07399, 0.0702, 0.06973, 0.07326, 0.07023, 0.06973]}, "backward-send-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [0.00026, 0.00021, 0.00019, 0.00019, 0.00019, 0.00018, 0.00019, 0.0002, 0.0002, 0.00019]}, "forward-send-backward-recv-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [6.16563, 0.28249, 0.27763, 0.28103, 0.27952, 0.28051, 0.2813, 0.28172, 0.29124, 0.28177]}, "backward-send-forward-recv-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [0.92523, 0.00228, 0.00214, 0.00215, 0.00226, 0.00213, 0.00217, 0.00235, 0.00224, 0.00219]}, "layernorm-grads-all-reduce-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [4e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05]}, "embedding-grads-all-reduce-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [6e-05, 3e-05, 3e-05, 3e-05, 3e-05, 4e-05, 3e-05, 3e-05, 3e-05, 4e-05]}, "all-grads-sync-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [0.19033, 0.00022, 0.00021, 0.00022, 0.00022, 0.00023, 0.00022, 0.00022, 0.00022, 0.00022]}, "optimizer-copy-to-main-grad-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [6e-05, 4e-05, 4e-05, 4e-05, 5e-05, 4e-05, 4e-05, 4e-05, 4e-05, 5e-05]}, "optimizer-clip-main-grad-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [2.24661, 0.00048, 0.00047, 0.00038, 0.00047, 0.00039, 0.00039, 0.00039, 0.00039, 0.0004]}, "optimizer-count-zeros-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [0.00926, 0.00069, 0.00062, 0.00063, 0.00063, 0.00063, 0.00062, 0.00063, 0.00062, 0.00062]}, "optimizer-inner-step-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [0.00112, 0.0002, 0.0002, 0.00021, 0.00021, 0.00021, 0.00021, 0.00021, 0.00022, 0.00021]}, "optimizer-copy-main-to-model-params-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [0.00014, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05]}, "optimizer-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [2.25814, 0.0021, 0.00203, 0.00193, 0.00201, 0.00193, 0.00195, 0.00196, 0.00197, 0.00195]}, "learning-rate": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, "learning-rate vs samples": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, "batch-size": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [32.0, 32.0, 32.0, 32.0, 32.0, 32.0, 32.0, 32.0, 32.0, 32.0]}, "batch-size vs samples": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [32.0, 32.0, 32.0, 32.0, 32.0, 32.0, 32.0, 32.0, 32.0, 32.0]}, "lm loss": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [9.19948, 9.20339, 9.20246, 9.19721, 9.1917, 9.18976, 9.18512, 9.17531, 9.17379, 9.16091]}, "lm loss vs samples": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [9.19948, 9.20339, 9.20246, 9.19721, 9.1917, 9.18976, 9.18512, 9.17531, 9.17379, 9.16091]}, "loss-scale": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, "loss-scale vs samples": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, "grad-norm": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [0.93282, 1.00192, 1.00046, 0.94405, 1.16906, 0.98576, 0.91648, 0.90421, 0.83062, 0.8822]}, "grad-norm vs samples": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [0.93282, 1.00192, 1.00046, 0.94405, 1.16906, 0.98576, 0.91648, 0.90421, 0.83062, 0.8822]}, "num-zeros": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [103.0, 122.0, 112.0, 97.0, 93.0, 105.0, 109.0, 107.0, 125.0, 130.0]}, "num-zeros vs samples": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [103.0, 122.0, 112.0, 97.0, 93.0, 105.0, 109.0, 107.0, 125.0, 130.0]}, "params-norm": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [35.67851, 35.67851, 35.67851, 35.67851, 35.67851, 35.67851, 35.67851, 35.6785, 35.67849, 35.67848]}, "params-norm vs samples": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [35.67851, 35.67851, 35.67851, 35.67851, 35.67851, 35.67851, 35.67851, 35.6785, 35.67849, 35.67848]}, "iteration-time": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [15.00501, 0.64144, 0.62022, 0.6193, 0.62312, 0.61981, 0.61869, 0.62693, 0.63288, 0.61782]}, "lm loss validation": {"start_step": 0, "end_step": 2, "step_interval": 5, "values": [9.15419]}, "lm loss validation vs samples": {"start_step": 0, "end_step": 2, "step_interval": 5, "values": [9.15419]}, "lm loss validation ppl": {"start_step": 0, "end_step": 2, "step_interval": 5, "values": [9453.99707]}, "lm loss validation ppl vs samples": {"start_step": 0, "end_step": 2, "step_interval": 5, "values": [9453.99707]}}
\ No newline at end of file