Fix #15898 (#15928)

examples/pytorch/language-modeling/run_clm.py

@@ -454,6 +454,10 @@ def main():
             eval_dataset = eval_dataset.select(range(data_args.max_eval_samples))
 
         def preprocess_logits_for_metrics(logits, labels):
+            if isinstance(logits, tuple):
+                # Depending on the model and config, logits may contain extra tensors,
+                # like past_key_values, but logits always come first
+                logits = logits[0]
             return logits.argmax(dim=-1)
 
         metric = load_metric("accuracy")

examples/pytorch/language-modeling/run_mlm.py

@@ -477,6 +477,10 @@ def main():
             eval_dataset = eval_dataset.select(range(data_args.max_eval_samples))
 
         def preprocess_logits_for_metrics(logits, labels):
+            if isinstance(logits, tuple):
+                # Depending on the model and config, logits may contain extra tensors,
+                # like past_key_values, but logits always come first
+                logits = logits[0]
             return logits.argmax(dim=-1)
 
         metric = load_metric("accuracy")