Merge pull request #568 from EleutherAI/revert-537-mmlu

Revert "Add MultipleChoiceExactTask"

lm_eval/base.py

@@ -743,20 +743,6 @@ class MultipleChoiceTask(Task):
         }
 
 
-class MultipleChoiceExactTask(MultipleChoiceTask):
-    def construct_requests(self, doc, ctx):
-        return rf.loglikelihood(ctx, self.doc_to_target(doc))[1]
-
-    def process_results(self, doc, results):
-        return {"acc": 1.0 if results[0] else 0.0}
-
-    def higher_is_better(self):
-        return {"acc": True}
-
-    def aggregation(self):
-        return {"acc": mean}
-
-
 class PerplexityTask(Task, abc.ABC):
     def should_decontaminate(self):
         """Whether this task supports decontamination against model training set."""