[chat] fix compute_approx_kl (#4338)

75c53890 · Wenhao Chen · GitHub · 03654c0c · 75c53890
Unverified Commit 75c53890 authored Aug 01, 2023 by Wenhao Chen Committed by GitHub Aug 01, 2023
Hide whitespace changes
Inline Side-by-side

Showing with 1 addition and 1 deletion

applications/Chat/coati/models/utils.py applications/Chat/coati/models/utils.py +1 -1

No files found.
--- a/applications/Chat/coati/models/utils.py
+++ b/applications/Chat/coati/models/utils.py
@@ -19,7 +19,7 @@ def compute_approx_kl(log_probs: torch.Tensor,
        action_mask: Mask for actions.
    """

-    log_ratio = log_probs - log_probs_base
+    log_ratio = log_probs_base - log_probs
    approx_kl = (log_ratio.exp() - 1) - log_ratio
    if action_mask is not None:
        approx_kl = masked_mean(approx_kl, action_mask, dim=1)