Merge pull request #766 from EleutherAI/patch-crowspairs

[Refactor] Patch crowspairs higher_is_better

lm_eval/tasks/crows_pairs/crows_pairs_english.yaml

@@ -16,6 +16,6 @@ metric_list:
   - metric: likelihood_diff
     aggregation: mean
     higher_is_better: false
-  - metric: acc
+  - metric: pct_stereotype
     aggregation: mean
-    higher_is_better: true
+    higher_is_better: false

lm_eval/tasks/crows_pairs/utils.py

@@ -13,7 +13,7 @@ def process_results(doc, results):
     # then treat this as predicting stereotyped sentence
     acc = 1.0 if likelihood1 > likelihood2 else 0.0
 
-    return {"likelihood_diff": diff, "acc": acc}
+    return {"likelihood_diff": diff, "pct_stereotype": acc}
 
 
 def doc_to_choice(doc):