Recent commit changed the bias dropout add jitted function which slightly...

Recent commit changed the bias dropout add jitted function which slightly changes numbers, manually tested to be accurate.

tests/functional_tests/test_results/bert/bert_tp1_pp2_1nodes_50steps.json

-{"lm loss": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [10.50444, 10.49325, 10.4863, 10.48386, 10.49859, 10.46608, 10.41875, 10.30048, 10.16226, 9.97872]}, "num-zeros": {"start_step": 0, "end_step": 34, "step_interval": 5, "values": [17438.0, 18790.0, 22561.0, 18532.0, 20246.0, 23670.0, 22788.0]}, "iteration_timing_avg": 0.3469323529411764}
+{"lm loss": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [10.50444, 10.49325, 10.4863, 10.48386, 10.49892, 10.46644, 10.41921, 10.30106, 10.16285, 9.97939]}, "num-zeros": {"start_step": 0, "end_step": 34, "step_interval": 5, "values": [17438.0, 18815.0, 22912.0, 18568.0, 19900.0, 23810.0, 22918.0]}, "iteration_timing_avg": 0.35970588235294115}
\ No newline at end of file

tests/functional_tests/test_results/bert/bert_tp1_pp4_1nodes_50steps.json

-{"lm loss": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [10.54369, 10.5383, 10.55953, 10.54011, 10.51871, 10.4908, 10.46566, 10.31844, 10.15596, 9.9664]}, "num-zeros": {"start_step": 0, "end_step": 34, "step_interval": 5, "values": [21736.0, 20410.0, 27256.0, 23697.0, 22528.0, 21048.0, 23461.0]}, "iteration_timing_avg": 0.8071679411764707}
+{"lm loss": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [10.54369, 10.5383, 10.55953, 10.54011, 10.51908, 10.49118, 10.46612, 10.31901, 10.15649, 9.96702]}, "num-zeros": {"start_step": 0, "end_step": 34, "step_interval": 5, "values": [21736.0, 20433.0, 27243.0, 23240.0, 22459.0, 20724.0, 23451.0]}, "iteration_timing_avg": 0.8657461764705884}
\ No newline at end of file

tests/functional_tests/test_results/bert/bert_tp2_pp2_1nodes_50steps.json

-{"lm loss": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [10.44729, 10.44093, 10.45375, 10.44445, 10.44267, 10.44555, 10.39114, 10.25849, 10.1345, 9.9564]}, "num-zeros": {"start_step": 0, "end_step": 34, "step_interval": 5, "values": [27334.0, 20715.0, 28442.0, 24309.0, 23479.0, 20540.0, 21108.0]}, "iteration_timing_avg": 0.618779411764706}
+{"lm loss": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [10.44729, 10.44093, 10.45375, 10.44445, 10.44305, 10.44595, 10.39163, 10.25898, 10.13498, 9.95692]}, "num-zeros": {"start_step": 0, "end_step": 34, "step_interval": 5, "values": [27334.0, 20551.0, 28114.0, 24328.0, 24070.0, 20653.0, 21346.0]}, "iteration_timing_avg": 0.6318655882352939}
\ No newline at end of file

tests/functional_tests/test_results/bert/bert_tp4_pp1_1nodes_50steps.json

-{"lm loss": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [10.4978, 10.49775, 10.48021, 10.50638, 10.49579, 10.46974, 10.34444, 10.25478, 10.10195, 9.91877]}, "num-zeros": {"start_step": 0, "end_step": 35, "step_interval": 5, "values": [26168.0, 19293.0, 28643.0, 22573.0, 25980.0, 34292.0, 21318.0]}, "iteration_timing_avg": 1.0391188235294118}
+{"lm loss": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [10.4978, 10.49775, 10.48021, 10.50638, 10.49624, 10.47018, 10.34494, 10.25536, 10.10244, 9.91938]}, "num-zeros": {"start_step": 0, "end_step": 35, "step_interval": 5, "values": [26168.0, 19042.0, 28718.0, 22408.0, 26377.0, 34320.0, 21873.0]}, "iteration_timing_avg": 1.1249785294117647}
\ No newline at end of file

tests/functional_tests/test_results/gpt3/gpt3_tp1_pp2_1nodes_50steps.json

-{"lm loss": {"start_step": 0, "end_step": 35, "step_interval": 5, "values": [10.84266, 10.89696, 10.90542, 10.87498, 10.86265, 10.83608, 10.64368]}, "num-zeros": {"start_step": 0, "end_step": 17, "step_interval": 5, "values": [2093.0, 2491.0, 2352.0, 2202.0]}, "iteration_timing_avg": 0.07941913043478262}
+{"lm loss": {"start_step": 0, "end_step": 37, "step_interval": 5, "values": [10.84266, 10.89696, 10.90542, 10.87498, 10.86279, 10.83628, 10.64437, 10.62386]}, "num-zeros": {"start_step": 0, "end_step": 20, "step_interval": 5, "values": [2093.0, 2474.0, 2327.0, 2213.0]}, "iteration_timing_avg": 0.080846}
\ No newline at end of file

tests/functional_tests/test_results/gpt3/gpt3_tp1_pp4_1nodes_50steps.json

-{"lm loss": {"start_step": 0, "end_step": 45, "step_interval": 5, "values": [10.7947, 10.85294, 10.87058, 10.83388, 10.83012, 10.78726, 10.56378, 10.57311, 10.48692]}, "num-zeros": {"start_step": 0, "end_step": 29, "step_interval": 5, "values": [2452.0, 2818.0, 2036.0, 2662.0, 2651.0, 2422.0]}, "iteration_timing_avg": 0.1187023333333333}
+{"lm loss": {"start_step": 0, "end_step": 49, "step_interval": 5, "values": [10.7947, 10.85294, 10.87058, 10.83388, 10.83025, 10.78755, 10.56419, 10.57339, 10.48735, 10.19553]}, "num-zeros": {"start_step": 0, "end_step": 33, "step_interval": 5, "values": [2452.0, 2744.0, 2176.0, 2722.0, 2636.0, 2535.0, 2996.0]}, "iteration_timing_avg": 0.1158709090909091}

tests/functional_tests/test_results/gpt3/gpt3_tp2_pp2_1nodes_50steps.json

-{"lm loss": {"start_step": 0, "end_step": 47, "step_interval": 5, "values": [10.85716, 10.88973, 10.879, 10.87014, 10.87964, 10.84443, 10.67203, 10.62868, 10.52733, 10.2536]}, "num-zeros": {"start_step": 0, "end_step": 30, "step_interval": 5, "values": [2450.0, 2383.0, 2525.0, 2234.0, 2313.0, 2514.0]}, "iteration_timing_avg": 0.11253562499999999}
+{"lm loss": {"start_step": 0, "end_step": 48, "step_interval": 5, "values": [10.85716, 10.88973, 10.879, 10.87014, 10.87978, 10.84463, 10.67266, 10.62932, 10.52767, 10.25362]}, "num-zeros": {"start_step": 0, "end_step": 31, "step_interval": 5, "values": [2450.0, 2396.0, 2523.0, 2242.0, 2225.0, 2478.0, 2536.0]}, "iteration_timing_avg": 0.11416968750000002}
\ No newline at end of file

tests/functional_tests/test_results/gpt3/gpt3_tp4_pp1_1nodes_50steps.json

-{"lm loss": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [10.86276, 10.88058, 10.87527, 10.88402, 10.89158, 10.84702, 10.6879, 10.62796, 10.53893, 10.26644]}, "num-zeros": {"start_step": 0, "end_step": 33, "step_interval": 5, "values": [2199.0, 2304.0, 2389.0, 1985.0, 2059.0, 2393.0, 2395.0]}, "iteration_timing_avg": 0.15685176470588238}
+{"lm loss": {"start_step": 0, "end_step": 50, "step_interval": 5, "values": [10.86276, 10.88058, 10.87527, 10.88402, 10.89173, 10.84724, 10.6886, 10.62864, 10.53925, 10.26646]}, "num-zeros": {"start_step": 0, "end_step": 33, "step_interval": 5, "values": [2199.0, 2306.0, 2412.0, 2032.0, 2077.0, 2475.0, 2347.0]}, "iteration_timing_avg": 0.15481029411764707}
\ No newline at end of file