Fix FP8 recompute by copying amaxes/scales to updated amaxes/scales instead of referencing (#1723)

Prevent precision issues by making get_old_fp8_meta_tensors_for_recompute() 
return copies of amaxes/scales rather than references to originals.
Signed-off-by: HuangHuang <huang.huang@mthreads.com>
Co-authored-by: Tim Moon <4406448+timmoon10@users.noreply.github.com>

transformer_engine/pytorch/fp8.py

@@ -513,8 +513,8 @@ class FP8GlobalStateManager:
             return
 
         # Store updated amaxes and scales from phase 1 post forward.
-        fp8_meta["updated_amax_history_fwd"] = fp8_meta["scaling_fwd"].amax_history
-        fp8_meta["updated_scale_fwd"] = fp8_meta["scaling_fwd"].scale
+        fp8_meta["updated_amax_history_fwd"] = fp8_meta["scaling_fwd"].amax_history.clone()
+        fp8_meta["updated_scale_fwd"] = fp8_meta["scaling_fwd"].scale.clone()
 
         # Retrieve stashed amaxes and scales from phase 1 pre forward.
         buffer_position_key = "global_fp8_buffer_pos_fwd_recompute"