update

lm_eval/tasks/squadv2/default.yaml

@@ -21,27 +21,27 @@ filter_list:
       - function: remove_whitespace
       - function: take_first
 metric_list:
-  - metric: !function metric.exact
-    aggregation: mean
-    higher_is_better: true
-  - metric: !function metric.f1
-    aggregation: mean
-    higher_is_better: true
-  - metric: !function metric.HasAns_exact
-    aggregation: mean
-    higher_is_better: true
-  - metric: !function metric.HasAns_f1
-    aggregation: mean
-    higher_is_better: true
-  - metric: !function metric.NoAns_exact
-    aggregation: mean
-    higher_is_better: true
-  - metric: !function metric.NoAns_f1
-    aggregation: mean
-    higher_is_better: true
-  - metric: !function metric.best_exact
-    aggregation: mean
-    higher_is_better: true
-  - metric: !function metric.best_f1
-    aggregation: mean
-    higher_is_better: true
+  - metric: exact
+    aggregation: !function utils.exact
+    higher_is_better: true
+  # - metric: f1
+  #   aggregation: mean
+  #   higher_is_better: true
+  # - metric: HasAns_exact
+  #   aggregation: mean
+  #   higher_is_better: true
+  # - metric: HasAns_f1
+  #   aggregation: mean
+  #   higher_is_better: true
+  # - metric: NoAns_exact
+  #   aggregation: mean
+  #   higher_is_better: true
+  # - metric: NoAns_f1
+  #   aggregation: mean
+  #   higher_is_better: true
+  # - metric: best_exact
+  #   aggregation: mean
+  #   higher_is_better: true
+  # - metric: best_f1
+  #   aggregation: mean
+  #   higher_is_better: true

lm_eval/tasks/squadv2/metric.py

-import evaluate
-from functools import partial
-
-
-def _squad_metric(predictions, references):
-    squad_metric = evaluate.load("squad_v2")
-    return squad_metric.compute(predictions=predictions, references=references)
-
-# Exact match (the normalized answer exactly match the gold answer)
-def exact(predictions, references):
-    return _squad_metric(predictions=predictions, references=references).get("exact", 0)
-
-# The F-score of predicted tokens versus the gold answer
-def f1(predictions, references):
-    return _squad_metric(predictions=predictions, references=references).get("f1", 0)
-
-# Exact match (the normalized answer exactly match the gold answer)
-def HasAns_exact(predictions, references):
-    return _squad_metric(predictions=predictions, references=references).get("HasAns_exact", 0)
-
-# The F-score of predicted tokens versus the gold answer
-def HasAns_f1(predictions, references):
-    return _squad_metric(predictions=predictions, references=references).get("HasAns_f1", 0)
-
-# Exact match (the normalized answer exactly match the gold answer)
-def NoAns_exact(predictions, references):
-    return _squad_metric(predictions=predictions, references=references).get("NoAns_exact", 0)
-
-# The F-score of predicted tokens versus the gold answer
-def NoAns_f1(predictions, references):
-    return _squad_metric(predictions=predictions, references=references).get("NoAns_f1", 0)
-
-# Best exact match (with varying threshold)
-def best_exact(predictions, references):
-    return _squad_metric(predictions=predictions, references=references).get("best_exact", 0)
-
-# Best F1 (with varying threshold)
-def best_f1(predictions, references):
-    return _squad_metric(predictions=predictions, references=references).get("best_f1", 0)

lm_eval/tasks/squadv2/utils.py

@@ -20,8 +20,12 @@ def process_results(doc, results):
         "answers": doc["answers"],
     }
 
-    print(_squad_metric(predictions, references))
-    return _squad_metric(predictions, references)
+    return {
+        "predictions": predictions,
+        "reference": references
+    }
+    # return _squad_metric([predictions], [references])
+    # return {key: value if key in metrics for key, value in score.items()}
 
 
 def _squad_metric(predictions, references):
@@ -29,33 +33,36 @@ def _squad_metric(predictions, references):
     return squad_metric.compute(predictions=predictions, references=references)
 
 # Exact match (the normalized answer exactly match the gold answer)
-def exact(predictions, references):
-    return _squad_metric(predictions=predictions, references=references).get("exact", 0)
+def exact(items):
+    print(items)
+    import sys; sys.exit()
+    predictions, references = zip(*items)
+    return _squad_metric(predictions=predictions, references=references)["exact"]
 
 # The F-score of predicted tokens versus the gold answer
 def f1(predictions, references):
-    return _squad_metric(predictions=predictions, references=references).get("f1", 0)
+    return _squad_metric(predictions=predictions, references=references)["f1"]
 
 # Exact match (the normalized answer exactly match the gold answer)
 def HasAns_exact(predictions, references):
-    return _squad_metric(predictions=predictions, references=references).get("HasAns_exact", 0)
+    return _squad_metric(predictions=predictions, references=references)["HasAns_exact"]
 
 # The F-score of predicted tokens versus the gold answer
 def HasAns_f1(predictions, references):
-    return _squad_metric(predictions=predictions, references=references).get("HasAns_f1", 0)
+    return _squad_metric(predictions=predictions, references=references)["HasAns_f1"]
 
 # Exact match (the normalized answer exactly match the gold answer)
 def NoAns_exact(predictions, references):
-    return _squad_metric(predictions=predictions, references=references).get("NoAns_exact", 0)
+    return _squad_metric(predictions=predictions, references=references)["NoAns_exact"]
 
 # The F-score of predicted tokens versus the gold answer
 def NoAns_f1(predictions, references):
-    return _squad_metric(predictions=predictions, references=references).get("NoAns_f1", 0)
+    return _squad_metric(predictions=predictions, references=references)["NoAns_f1"]
 
 # Best exact match (with varying threshold)
 def best_exact(predictions, references):
-    return _squad_metric(predictions=predictions, references=references).get("best_exact", 0)
+    return _squad_metric(predictions=predictions, references=references)["best_exact"]
 
 # Best F1 (with varying threshold)
 def best_f1(predictions, references):
-    return _squad_metric(predictions=predictions, references=references).get("best_f1", 0)
+    return _squad_metric(predictions=predictions, references=references)["best_f1"]