Merge pull request #80 from nicholaskross/master

Started SAT eval

lm_eval/tasks/sat.py

-# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
-
-import json
-import random
-import os
-from lm_eval.base import Dataset
-from ..utils import sh
-
-
-class SATAnalogies(Dataset):    
-    def __init__(self):
-        super().__init__()
-
-    def download(self):
-        # We should be using a checksum here.
-        # The canonical sha256 hash is below:
-        # 9dece377d8d57253ef8c78370ff15de0bb1d9e90a82c815a67ba1e621e921bfc
-        if not os.path.exists('data/sat') and os.path.exists('data/sat/SAT-package-V3.txt'):
-            raise NotImplementedError('SAT Analogies dataset is not provided. Follow instructions on https://aclweb.org/aclwiki/SAT_Analogy_Questions_(State_of_the_art) to locate.')
-
-    def has_training_docs(self):
-        return False
-
-    def has_validation_docs(self):
-        return False
-
-    def has_test_docs(self):
-        return True
-
-    def training_docs(self):
-        return []
-
-    def validation_docs(self):
-        return []
-
-    def test_docs(self):
-        data = []
-
-        with open("data/sat/SAT-package-V3.txt", "r") as f:
-            lines = f.read().splitlines() 
-            record = []
-            for line in lines:
-                if len(line) == 0 and record:
-                    data.append(record)
-                    record = []
-                elif len(line) > 0 and line[0] == '#':
-                    continue
-                else:
-                    record.append(line)
-            data.append(record)
-
-        docs = []
-
-        for record in data:
-            source = record[-8]
-            query = record[-7]
-            choices = record[-6:-1]
-            answer_key = record[-1]
-
-            doc = {
-                'source': source,
-                'query': query,
-                'choices': choices,
-                'answer_key': answer_key,
-            }
-            docs.append(doc)
-
-        return docs
-
-    
-    def fewshot_description(self):
-        # This format is ONLY for the purposes of deduplication. For the task evaluation, we'll need to find a new strategy,
-        # to meet the needs of this particular task.
-        return "first thing is to second thing as\nthird thing is to fourth thing\nfifth thing is to sixth thing\nseventh thing is to eighth thing\nninth thing is to tenth thing\neleventh thing is to twelfth thing\nanswer which is either a b c d or e"
-
-    def doc_to_text(self, doc, include_target=True):
-        # SAT Analogies is currently only writing out full examples. Partial evaluation needs implementing.
-        format_qn = lambda x: x[0] + ' is to ' + x[1]
-
-        query = doc['query']
-        choices = doc['choices']
-        answer = doc['answer_key']
-
-        query_words = query.split(' ')[:2]
-        text = format_qn(query_words) + ' as' + '\n'
-
-        for choice in choices:
-            choice_words = choice.split(' ')[:2]
-            text += format_qn(choice_words) + '\n'
-
-        if include_target:
-            text += answer
-
-        return text
-
-    # TODO: Implement evaluation code
-
-    # ***IMPORTANT***: this evaluation function needs to be written for the new framework. 
-    # For more info, check out the interface in base.py and the example BoolQ implementation in superglue.py. 
-    # Remove this comment when the evaluation code is implemented.
\ No newline at end of file
+# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
+
+import json
+import random
+import os
+from lm_eval.base import Dataset, rf, mean
+from tqdm import auto as tqdm_lib
+from . common import simple_accuracy_metric
+import numpy as np
+from ..utils import sh
+
+
+class SATAnalogies(Dataset):    
+    def __init__(self):
+        super().__init__()
+
+    def download(self):
+        # We should be using a checksum here.
+        # The canonical sha256 hash is below:
+        # 9dece377d8d57253ef8c78370ff15de0bb1d9e90a82c815a67ba1e621e921bfc
+        if not os.path.exists('data/sat') and os.path.exists('data/sat/SAT-package-V3.txt'):
+            raise NotImplementedError('SAT Analogies dataset is not provided. Follow instructions on https://aclweb.org/aclwiki/SAT_Analogy_Questions_(State_of_the_art) to locate.')
+
+    def has_training_docs(self):
+        return False
+
+    def has_validation_docs(self):
+        return False
+
+    def has_test_docs(self):
+        return True
+
+    def training_docs(self):
+        return []
+
+    def validation_docs(self):
+        return []
+
+    def test_docs(self):
+        data = []
+
+        with open("data/sat/SAT-package-V3.txt", "r") as f:
+            lines = f.read().splitlines() 
+            record = []
+            for line in lines:
+                if len(line) == 0 and record:
+                    data.append(record)
+                    record = []
+                elif len(line) > 0 and line[0] == '#':
+                    continue
+                else:
+                    record.append(line)
+            data.append(record)
+
+        docs = []
+
+        for record in data:
+            source = record[-8]
+            query = record[-7]
+            choices = record[-6:-1]
+            answer_key = record[-1]
+
+            doc = {
+                'source': source,
+                'query': query,
+                'choices': choices,
+                'answer_key': answer_key,
+            }
+            docs.append(doc)
+
+        return docs
+
+    
+    def fewshot_description(self):
+        # This format is ONLY for the purposes of deduplication. For the task evaluation, we'll need to find a new strategy,
+        # to meet the needs of this particular task.
+        return "first thing is to second thing as\nthird thing is to fourth thing\nfifth thing is to sixth thing\nseventh thing is to eighth thing\nninth thing is to tenth thing\neleventh thing is to twelfth thing\nanswer which is either a b c d or e"
+
+    def doc_to_text(self, doc, include_target=True):
+        # SAT Analogies is currently only writing out full examples. Partial evaluation needs implementing.
+        format_qn = lambda x: x[0] + ' is to ' + x[1]
+
+        query = doc['query']
+        choices = doc['choices']
+        answer = doc['answer_key']
+
+        query_words = query.split(' ')[:2]
+        text = format_qn(query_words) + ' as' + '\n'
+
+        for choice in choices:
+            choice_words = choice.split(' ')[:2]
+            text += format_qn(choice_words) + '\n'
+
+        if include_target:
+            text += answer
+
+        return text
+
+
+    def doc_to_target(self, doc):
+        # assumes answer_key is the true-answer's letter
+        return doc['answer_key']
+
+    def construct_requests(self, ctx):
+        # assumes the output is the predicted-answer's letter
+        ll_a = rf.loglikelihood(ctx, ' a')
+        ll_b = rf.loglikelihood(ctx, ' b')
+        ll_c = rf.loglikelihood(ctx, ' c')
+        ll_d = rf.loglikelihood(ctx, ' d')
+        ll_e = rf.loglikelihood(ctx, ' e')
+
+        return ll_a, ll_b, ll_c, ll_d, ll_e
+
+    def process_results(self, doc, results):
+        predicted_odds = np.array(list(results))
+        gold = doc["answer_key"]
+
+        acc = 1. if np.argmax(predicted_odds) == gold else 0.
+
+        return [
+            {
+                "submetric": "acc",
+                "value": acc,
+                "higher_is_better": True,
+                "aggregation": mean
+            }
+        ]
+
+
+    def evaluate(self, docs, lm):
+        # functionality already implemented above
+        raise NotImplementedError()