load metric with `evaluate` (#2351)

f378f306 · Baber Abbasi · GitHub · bc50a9aa · f378f306
Unverified Commit f378f306 authored Sep 26, 2024 by Baber Abbasi Committed by GitHub Sep 25, 2024
Hide whitespace changes
Inline Side-by-side

Showing with 3 additions and 1 deletion

lm_eval/tasks/squadv2/task.py lm_eval/tasks/squadv2/task.py +3 -1

No files found.
--- a/lm_eval/tasks/squadv2/task.py
+++ b/lm_eval/tasks/squadv2/task.py
@@ -37,7 +37,9 @@ _CITATION = """


 def _squad_metric(predictions, references):
-    squad_metric = datasets.load_metric("squad_v2")
+    import evaluate
+
+    squad_metric = evaluate.load("squad_v2")
    return squad_metric.compute(predictions=predictions, references=references)