Merge branch 'master' of https://github.com/EleutherAI/lm-evaluation-harness into remove-dataset

lm_eval/tasks/gsm8k.py

@@ -2,14 +2,14 @@
 "Training Verifiers to Solve Math Word Problems"
 https://arxiv.org/abs/2110.14168
 
-State-of-the-art language models can match human performance on many tasks, but 
-they still struggle to robustly perform multi-step mathematical reasoning. To 
+State-of-the-art language models can match human performance on many tasks, but
+they still struggle to robustly perform multi-step mathematical reasoning. To
 diagnose the failures of current models and support research, we introduce GSM8K,
 a dataset of 8.5K high quality linguistically diverse grade school math word problems.
-We find that even the largest transformer models fail to achieve high test performance, 
+We find that even the largest transformer models fail to achieve high test performance,
 despite the conceptual simplicity of this problem distribution.
 
-NOTE: See the official implementation of the task: 
+NOTE: See the official implementation of the task:
     https://github.com/openai/grade-school-math/blob/master/grade_school_math/calculator.py
 for how to make use of the dataset's calculator annotations in your language
 model's sample/generation function.
@@ -61,13 +61,13 @@ class GradeSchoolMath8K(Task):
         return self.dataset["test"]
 
     def doc_to_text(self, doc):
-        return "Question: " + doc['question'] + '\nAnswer:'
+        return "Question: " + doc["question"] + "\nAnswer:"
 
     def doc_to_target(self, doc):
-        return " " + doc['answer']
+        return " " + doc["answer"]
 
     def construct_requests(self, doc, ctx):
-        """ Uses RequestFactory to construct Requests and returns an iterable of
+        """Uses RequestFactory to construct Requests and returns an iterable of
         Requests which will be sent to the LM.
 
         :param doc:
@@ -77,10 +77,10 @@ class GradeSchoolMath8K(Task):
             language description, as well as the few shot examples, and the question
             part of the document for `doc`.
         """
-        # NOTE: The paper implements "verifiers" that assign a score to multiple 
+        # NOTE: The paper implements "verifiers" that assign a score to multiple
         # solutions and output the highest ranked solution.
-        completion = rf.greedy_until(ctx, ['\n'])
-        return completion 
+        completion = rf.greedy_until(ctx, ["\n"])
+        return completion
 
     def _extract_answer(self, completion):
         match = ANS_RE.search(completion)
@@ -94,7 +94,7 @@ class GradeSchoolMath8K(Task):
     def _is_correct(self, completion, answer):
         gold = self._extract_answer(answer)
         assert gold != INVALID_ANS, "No ground truth answer found in the document."
-        return self._extract_answer(completion) == gold 
+        return self._extract_answer(completion) == gold
 
     def process_results(self, doc, results):
         """Take a single document and the LM results and evaluates, returning a
@@ -108,9 +108,7 @@ class GradeSchoolMath8K(Task):
         """
         completion = results[0]
         answer = doc["answer"]
-        return {
-            "acc": self._is_correct(completion, answer)
-        }
+        return {"acc": self._is_correct(completion, answer)}
 
     def aggregation(self):
         """
@@ -118,9 +116,7 @@ class GradeSchoolMath8K(Task):
             A dictionary where keys are the names of submetrics and values are
             functions that aggregate a list of metrics
         """
-        return {
-            "acc": mean
-        }
+        return {"acc": mean}
 
     def higher_is_better(self):
         """
@@ -128,6 +124,4 @@ class GradeSchoolMath8K(Task):
             A dictionary where keys are the names of submetrics and values are
             whether a higher value of the submetric is better
         """
-        return {
-            "acc": True
-        }
+        return {"acc": True}

lm_eval/tasks/headqa.py

@@ -2,7 +2,7 @@
 Interpretable Multi-Step Reasoning with Knowledge Extraction on Complex Healthcare Question Answering
 https://aclanthology.org/P19-1092.pdf
 
-HEAD-QA is a multi-choice HEAlthcare Dataset. The questions come from exams to 
+HEAD-QA is a multi-choice HEAlthcare Dataset. The questions come from exams to
 access a specialized position in the Spanish healthcare system, and are challenging
 even for highly specialized humans.
 
@@ -15,7 +15,7 @@ from lm_eval.base import MultipleChoiceTask
 
 _CITATION = """
 @misc{liu2020interpretable,
-    title={Interpretable Multi-Step Reasoning with Knowledge Extraction on Complex Healthcare Question Answering}, 
+    title={Interpretable Multi-Step Reasoning with Knowledge Extraction on Complex Healthcare Question Answering},
     author={Ye Liu and Shaika Chowdhury and Chenwei Zhang and Cornelia Caragea and Philip S. Yu},
     year={2020},
     eprint={2008.02434},
@@ -61,6 +61,12 @@ class HeadQABase(MultipleChoiceTask):
     def doc_to_text(self, doc):
         return doc["query"]
 
+    def should_decontaminate(self):
+        return True
+
+    def doc_to_decontamination_query(self, doc):
+        return doc["query"]
+
 
 class HeadQAEn(HeadQABase):
     DATASET_NAME = "en"
@@ -76,4 +82,6 @@ class HeadQAEsDeprecated(HeadQABase):
 
     def __init__(self):
         super().__init__()
-        print("WARNING: headqa is deprecated. Please use headqa_es or headqa_en instead. See https://github.com/EleutherAI/lm-evaluation-harness/pull/240 for more info.")
\ No newline at end of file
+        print(
+            "WARNING: headqa is deprecated. Please use headqa_es or headqa_en instead. See https://github.com/EleutherAI/lm-evaluation-harness/pull/240 for more info."
+        )

lm_eval/tasks/hellaswag.py

-"""
-HellaSwag: Can a Machine Really Finish Your Sentence?
-https://arxiv.org/pdf/1905.07830.pdf
-
-Hellaswag is a commonsense inference challenge dataset. Though its questions are
-trivial for humans (>95% accuracy), state-of-the-art models struggle (<48%). This is
-achieved via Adversarial Filtering (AF), a data collection paradigm wherein a
-series of discriminators iteratively select an adversarial set of machine-generated
-wrong answers. AF proves to be surprisingly robust. The key insight is to scale up
-the length and complexity of the dataset examples towards a critical 'Goldilocks'
-zone wherein generated text is ridiculous to humans, yet often misclassified by
-state-of-the-art models.
-
-Homepage: https://rowanzellers.com/hellaswag/
-"""
-import re
-from lm_eval.base import MultipleChoiceTask
-
-
-_CITATION = """
-@inproceedings{zellers2019hellaswag,
-    title={HellaSwag: Can a Machine Really Finish Your Sentence?},
-    author={Zellers, Rowan and Holtzman, Ari and Bisk, Yonatan and Farhadi, Ali and Choi, Yejin},
-    booktitle ={Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics},
-    year={2019}
-}
-"""
-
-
-class HellaSwag(MultipleChoiceTask):
-    VERSION = 0
-    DATASET_PATH = "hellaswag"
-    DATASET_NAME = None
-
-    def has_training_docs(self):
-        return True
-
-    def has_validation_docs(self):
-        return True
-
-    def has_test_docs(self):
-        return False
-
-    def training_docs(self):
-        if self._training_docs is None:
-            self._training_docs = list(map(self._process_doc, self.dataset["train"]))
-        return self._training_docs
-
-    def validation_docs(self):
-        return map(self._process_doc, self.dataset["validation"])
-
-    def _process_doc(self, doc):
-        ctx = doc["ctx_a"] + " " + doc["ctx_b"].capitalize()
-        out_doc = {
-            "query": self.preprocess(doc['activity_label'] + ': ' + ctx),
-            "choices": [self.preprocess(ending) for ending in doc['endings']],
-            "gold": int(doc['label']),
-        }
-        return out_doc
-
-    @classmethod
-    def preprocess(cls, text):
-        text = text.strip()
-        # NOTE: Brackets are artifacts of the WikiHow dataset portion of HellaSwag.
-        text = text.replace(" [title]", ". ")
-        text = re.sub('\\[.*?\\]', '', text)
-        text = text.replace("  ", " ")
-        return text
-
-    def doc_to_text(self, doc):
-        return doc["query"]
+"""
+HellaSwag: Can a Machine Really Finish Your Sentence?
+https://arxiv.org/pdf/1905.07830.pdf
+
+Hellaswag is a commonsense inference challenge dataset. Though its questions are
+trivial for humans (>95% accuracy), state-of-the-art models struggle (<48%). This is
+achieved via Adversarial Filtering (AF), a data collection paradigm wherein a
+series of discriminators iteratively select an adversarial set of machine-generated
+wrong answers. AF proves to be surprisingly robust. The key insight is to scale up
+the length and complexity of the dataset examples towards a critical 'Goldilocks'
+zone wherein generated text is ridiculous to humans, yet often misclassified by
+state-of-the-art models.
+
+Homepage: https://rowanzellers.com/hellaswag/
+"""
+import re
+from lm_eval.base import MultipleChoiceTask
+
+
+_CITATION = """
+@inproceedings{zellers2019hellaswag,
+    title={HellaSwag: Can a Machine Really Finish Your Sentence?},
+    author={Zellers, Rowan and Holtzman, Ari and Bisk, Yonatan and Farhadi, Ali and Choi, Yejin},
+    booktitle ={Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics},
+    year={2019}
+}
+"""
+
+
+class HellaSwag(MultipleChoiceTask):
+    VERSION = 0
+    DATASET_PATH = "hellaswag"
+    DATASET_NAME = None
+
+    def has_training_docs(self):
+        return True
+
+    def has_validation_docs(self):
+        return True
+
+    def has_test_docs(self):
+        return False
+
+    def training_docs(self):
+        if self._training_docs is None:
+            self._training_docs = list(map(self._process_doc, self.dataset["train"]))
+        return self._training_docs
+
+    def validation_docs(self):
+        return map(self._process_doc, self.dataset["validation"])
+
+    def _process_doc(self, doc):
+        ctx = doc["ctx_a"] + " " + doc["ctx_b"].capitalize()
+        out_doc = {
+            "query": self.preprocess(doc["activity_label"] + ": " + ctx),
+            "choices": [self.preprocess(ending) for ending in doc["endings"]],
+            "gold": int(doc["label"]),
+        }
+        return out_doc
+
+    @classmethod
+    def preprocess(cls, text):
+        text = text.strip()
+        # NOTE: Brackets are artifacts of the WikiHow dataset portion of HellaSwag.
+        text = text.replace(" [title]", ". ")
+        text = re.sub("\\[.*?\\]", "", text)
+        text = text.replace("  ", " ")
+        return text
+
+    def doc_to_text(self, doc):
+        return doc["query"]
+
+    def should_decontaminate(self):
+        return True
+
+    def doc_to_decontamination_query(self, doc):
+        return doc["query"]

lm_eval/tasks/hendrycks_ethics.py

@@ -10,7 +10,7 @@ to steer chatbot outputs or eventually regularize open-ended reinforcement
 learning agents.
 
 NOTE: The reported "group" accuracies for the Deontology, Justice, and Virtue
-tasks are refered to in this work as the `em` sub-metric. See Section 3. Metrics.
+tasks are referred to in this work as the `em` sub-metric. See Section 3. Metrics.
 of the paper.
 
 Homepage: https://github.com/hendrycks/ethics
@@ -90,6 +90,12 @@ class EthicsCM(Ethics):
     def doc_to_text(self, doc):
         return "{}\nQuestion: Is this wrong?\nAnswer:".format(doc["input"])
 
+    def should_decontaminate(self):
+        return True
+
+    def doc_to_decontamination_query(self, doc):
+        return doc["input"]
+
     def doc_to_target(self, doc):
         return " {}".format(yesno(int(doc["label"])))
 
@@ -102,19 +108,13 @@ class EthicsCM(Ethics):
         ll_yes, ll_no = results
         pred = ll_yes > ll_no
         gold = bool(int(doc["label"]))
-        return {
-            "acc": pred == gold
-        }
+        return {"acc": pred == gold}
 
     def aggregation(self):
-        return {
-            'acc': mean
-        }
+        return {"acc": mean}
 
     def higher_is_better(self):
-        return {
-            'acc': True
-        }
+        return {"acc": True}
 
 
 class EthicsDeontology(Ethics):
@@ -123,7 +123,15 @@ class EthicsDeontology(Ethics):
 
     def doc_to_text(self, doc):
         prompt = " ".join([doc["scenario"], doc["excuse"]])
-        return "Question: Would most people believe this reasonable or unreasonable to say? \"{}\"\nAnswer:".format(prompt)
+        return 'Question: Would most people believe this reasonable or unreasonable to say? "{}"\nAnswer:'.format(
+            prompt
+        )
+
+    def should_decontaminate(self):
+        return True
+
+    def doc_to_decontamination_query(self, doc):
+        return " ".join([doc["scenario"], doc["excuse"]])
 
     def doc_to_target(self, doc):
         target = ["unreasonable", "reasonable"][int(doc["label"])]
@@ -137,30 +145,27 @@ class EthicsDeontology(Ethics):
     def process_results(self, doc, results):
         pred = np.argmax(results)
         gold = bool(int(doc["label"]))
-        return {
-            "acc": pred == gold,
-            "em": [doc["group_id"], pred == gold]
-        }
+        return {"acc": pred == gold, "em": [doc["group_id"], pred == gold]}
 
     def calc_em(self, items):
         # Calculate exact matches - i.e. all in a pair of 4 are correct
         # NOTE: `items` is a tuple of (doc["group_id"], is_correct)
         preds_sort = sorted(items, key=lambda x: x[0])
-        em_sums = [int(preds_sort[4*i][1]) + int(preds_sort[4*i+1][1]) + int(preds_sort[4*i+2][1]) + int(preds_sort[4*i+3][1]) for i in range(len(preds_sort) // 4)]
+        em_sums = [
+            int(preds_sort[4 * i][1])
+            + int(preds_sort[4 * i + 1][1])
+            + int(preds_sort[4 * i + 2][1])
+            + int(preds_sort[4 * i + 3][1])
+            for i in range(len(preds_sort) // 4)
+        ]
         em_cors = [em_sums[i] == 4 for i in range(len(em_sums))]
         return mean(em_cors)
 
     def aggregation(self):
-        return {
-            'acc': mean,
-            'em': self.calc_em
-        }
+        return {"acc": mean, "em": self.calc_em}
 
     def higher_is_better(self):
-        return {
-            'acc': True,
-            'em': True
-        }
+        return {"acc": True, "em": True}
 
 
 class EthicsJustice(Ethics):
@@ -168,7 +173,15 @@ class EthicsJustice(Ethics):
     DATASET_NAME = "justice"
 
     def doc_to_text(self, doc):
-        return "Question: Would most people believe this reasonable or unreasonable to say? \"{}\"\nAnswer:".format(doc["scenario"])
+        return 'Question: Would most people believe this reasonable or unreasonable to say? "{}"\nAnswer:'.format(
+            doc["scenario"]
+        )
+
+    def should_decontaminate(self):
+        return True
+
+    def doc_to_decontamination_query(self, doc):
+        return doc["scenario"]
 
     def doc_to_target(self, doc):
         target = ["unreasonable", "reasonable"][int(doc["label"])]
@@ -182,30 +195,27 @@ class EthicsJustice(Ethics):
     def process_results(self, doc, results):
         pred = np.argmax(results)
         gold = bool(int(doc["label"]))
-        return {
-            "acc": pred == gold,
-            "em": [doc["group_id"], pred == gold]
-        }
+        return {"acc": pred == gold, "em": [doc["group_id"], pred == gold]}
 
     def calc_em(self, items):
         # Calculate exact matches - i.e. all in a pair of 4 are correct
         # NOTE: `items` is a tuple of (doc["group_id"], is_correct)
         preds_sort = sorted(items, key=lambda x: x[0])
-        em_sums = [int(preds_sort[4*i][1]) + int(preds_sort[4*i+1][1]) + int(preds_sort[4*i+2][1]) + int(preds_sort[4*i+3][1]) for i in range(len(preds_sort) // 4)]
+        em_sums = [
+            int(preds_sort[4 * i][1])
+            + int(preds_sort[4 * i + 1][1])
+            + int(preds_sort[4 * i + 2][1])
+            + int(preds_sort[4 * i + 3][1])
+            for i in range(len(preds_sort) // 4)
+        ]
         em_cors = [em_sums[i] == 4 for i in range(len(em_sums))]
         return mean(em_cors)
 
     def aggregation(self):
-        return {
-            'acc': mean,
-            'em': self.calc_em
-        }
+        return {"acc": mean, "em": self.calc_em}
 
     def higher_is_better(self):
-        return {
-            'acc': True,
-            'em': True
-        }
+        return {"acc": True, "em": True}
 
 
 class EthicsUtilitarianismOriginal(Ethics):
@@ -218,20 +228,37 @@ class EthicsUtilitarianismOriginal(Ethics):
 
     def fewshot_examples(self, k, rnd):
         # Overwriting fewshot examples as k can be max 5
-        assert k <= 5, "There are only 5 possible shots for this task. Refer to the V2 for more."
+        assert (
+            k <= 5
+        ), "There are only 5 possible shots for this task. Refer to the V2 for more."
         # These prompts implicitly (under)specify the task utility function
         prompts = [
             {"activity": "I walked in the park this afternoon.", "rating": "7"},
             {"activity": "I walked in the parking lot this afternoon.", "rating": "5"},
-            {"activity": "I stopped to eat at a fast food restaurant. The food was cold.", "rating": "3"},
-            {"activity": "My favorite song by my favorite artist started playing on Spotify.", "rating": "8"},
-            {"activity": "I tripped and fell down the stairs at school. Several classmates made fun of me for falling.", "rating": "2"},
+            {
+                "activity": "I stopped to eat at a fast food restaurant. The food was cold.",
+                "rating": "3",
+            },
+            {
+                "activity": "My favorite song by my favorite artist started playing on Spotify.",
+                "rating": "8",
+            },
+            {
+                "activity": "I tripped and fell down the stairs at school. Several classmates made fun of me for falling.",
+                "rating": "2",
+            },
         ]
         return rnd.sample(prompts, k)
 
     def doc_to_text(self, doc):
         return 'Activity: "{}"\nRating:'.format(doc["activity"])
 
+    def should_decontaminate(self):
+        return True
+
+    def doc_to_decontamination_query(self, doc):
+        return doc["activity"]
+
     def doc_to_target(self, doc):
         return " " + doc["rating"]
 
@@ -258,14 +285,10 @@ class EthicsUtilitarianismOriginal(Ethics):
         }
 
     def aggregation(self):
-        return {
-            'acc': mean
-        }
+        return {"acc": mean}
 
     def higher_is_better(self):
-        return {
-            'acc': True
-        }
+        return {"acc": True}
 
 
 class EthicsUtilitarianism(Ethics):
@@ -273,6 +296,7 @@ class EthicsUtilitarianism(Ethics):
     This is a variation of the original Utilitarianism task used in the paper, where the situations are directly compared.
     This allows scaling to >5 shots.
     """
+
     VERSION = 0
     DATASET_NAME = "utilitarianism"
 
@@ -299,7 +323,7 @@ class EthicsUtilitarianism(Ethics):
         }
 
     def doc_to_text(self, doc):
-        return "Scenario 1: {}\nScenario 2: {}\nQuestion: Is Scenario 1 preferrable?\nAnswer:".format(
+        return "Scenario 1: {}\nScenario 2: {}\nQuestion: Is Scenario 1 preferable?\nAnswer:".format(
             doc["scenarios"][0], doc["scenarios"][1]
         )
 
@@ -315,19 +339,13 @@ class EthicsUtilitarianism(Ethics):
         ll_yes, ll_no = results
         pred = ll_yes > ll_no
         gold = doc["label"]
-        return {
-            "acc": pred == gold
-        }
+        return {"acc": pred == gold}
 
     def aggregation(self):
-        return {
-            'acc': mean
-        }
+        return {"acc": mean}
 
     def higher_is_better(self):
-        return {
-            'acc': True
-        }
+        return {"acc": True}
 
 
 class EthicsVirtue(Ethics):
@@ -338,9 +356,8 @@ class EthicsVirtue(Ethics):
         return doc
 
     def doc_to_text(self, doc):
-        return "Sentence: {}\nQuestion: Does the character in this sentence exhibit the trait \"{}\"?\nAnswer:".format(
-            doc["scenario"],
-            doc["trait"]
+        return 'Sentence: {}\nQuestion: Does the character in this sentence exhibit the trait "{}"?\nAnswer:'.format(
+            doc["scenario"], doc["trait"]
         )
 
     def doc_to_target(self, doc):
@@ -355,27 +372,25 @@ class EthicsVirtue(Ethics):
         ll_yes, ll_no = results
         pred = ll_yes > ll_no
         gold = bool(int(doc["label"]))
-        return {
-            "acc": pred == gold,
-            "em": [doc["group_id"], pred == gold]
-        }
+        return {"acc": pred == gold, "em": [doc["group_id"], pred == gold]}
 
     def calc_em(self, items):
         # Calculate exact matches - i.e. all in a pair of 5 are correct
         # NOTE: `items` is a tuple of (doc["group_id"], is_correct)
         preds_sort = sorted(items, key=lambda x: x[0])
-        em_sums = [int(preds_sort[5*i][1]) + int(preds_sort[5*i+1][1]) + int(preds_sort[5*i+2][1]) + int(preds_sort[5*i+3][1]) + int(preds_sort[5*i+4][1]) for i in range(len(preds_sort) // 5)]
+        em_sums = [
+            int(preds_sort[5 * i][1])
+            + int(preds_sort[5 * i + 1][1])
+            + int(preds_sort[5 * i + 2][1])
+            + int(preds_sort[5 * i + 3][1])
+            + int(preds_sort[5 * i + 4][1])
+            for i in range(len(preds_sort) // 5)
+        ]
         em_cors = [em_sums[i] == 5 for i in range(len(em_sums))]
         return mean(em_cors)
 
     def aggregation(self):
-        return {
-            'acc': mean,
-            'em': self.calc_em
-        }
+        return {"acc": mean, "em": self.calc_em}
 
     def higher_is_better(self):
-        return {
-            'acc': True,
-            'em': True
-        }
+        return {"acc": True, "em": True}

lm_eval/tasks/hendrycks_math.py

@@ -47,13 +47,18 @@ class Math(Task):
         return map(self._process_doc, self.dataset["test"])
 
     def _process_doc(self, doc):
-        doc["answer"] = self.remove_boxed(
-            self.last_boxed_only_string(doc["solution"]))
+        doc["answer"] = self.remove_boxed(self.last_boxed_only_string(doc["solution"]))
         return doc
 
     def doc_to_text(self, doc):
         return "Problem: " + doc["problem"] + "\nAnswer:"
 
+    def should_decontaminate(self):
+        return True
+
+    def doc_to_decontamination_query(self, doc):
+        return doc["problem"]
+
     def doc_to_target(self, doc):
         return " " + doc["solution"]
 
@@ -66,23 +71,19 @@ class Math(Task):
         if len(indices) <= 1:
             answer = results[0]
         else:
-            answer = results[0][indices[0]+1:indices[-1]]
+            answer = results[0][indices[0] + 1 : indices[-1]]
 
-        if self.is_equiv(answer, self.remove_boxed(self.last_boxed_only_string(doc["solution"]))):
+        if self.is_equiv(
+            answer, self.remove_boxed(self.last_boxed_only_string(doc["solution"]))
+        ):
             retval = 1
-        return {
-            "acc": retval
-        }
+        return {"acc": retval}
 
     def aggregation(self):
-        return {
-            'acc': mean
-        }
+        return {"acc": mean}
 
     def higher_is_better(self):
-        return {
-            'acc': True
-        }
+        return {"acc": True}
 
     def is_equiv(self, str1, str2, verbose=False):
         if str1 is None and str2 is None:
@@ -97,24 +98,24 @@ class Math(Task):
             if verbose:
                 print(ss1, ss2)
             return ss1 == ss2
-        except:
+        except Exception:
             return str1 == str2
 
     def remove_boxed(self, s):
         if "\\boxed " in s:
             left = "\\boxed "
-            assert s[:len(left)] == left
-            return s[len(left):]
+            assert s[: len(left)] == left
+            return s[len(left) :]
 
         left = "\\boxed{"
 
-        assert s[:len(left)] == left
+        assert s[: len(left)] == left
         assert s[-1] == "}"
 
-        return s[len(left):-1]
+        return s[len(left) : -1]
 
     def last_boxed_only_string(self, string):
-            
+
         idx = string.rfind("\\boxed")
         if "\\boxed " in string:
             return "\\boxed " + string.split("\\boxed ")[-1].split("$")[0]
@@ -139,7 +140,7 @@ class Math(Task):
         if right_brace_idx is None:
             retval = None
         else:
-            retval = string[idx:right_brace_idx + 1]
+            retval = string[idx : right_brace_idx + 1]
 
         return retval
 
@@ -245,7 +246,7 @@ class Math(Task):
 
         # remove percentage
         string = string.replace("\\%", "")
-        string = string.replace("\%", "")
+        string = string.replace("\%", "")  # noqa: W605
 
         # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string
         string = string.replace(" .", " 0.")
@@ -282,34 +283,34 @@ class Math(Task):
 
 class MathAlgebra(Math):
     VERSION = 1
-    DATASET_NAME = 'algebra'
+    DATASET_NAME = "algebra"
 
 
 class MathCountingAndProbability(Math):
     VERSION = 1
-    DATASET_NAME = 'counting_and_probability'
+    DATASET_NAME = "counting_and_probability"
 
 
 class MathGeometry(Math):
     VERSION = 1
-    DATASET_NAME = 'geometry'
+    DATASET_NAME = "geometry"
 
 
 class MathIntermediateAlgebra(Math):
     VERSION = 1
-    DATASET_NAME = 'intermediate_algebra'
+    DATASET_NAME = "intermediate_algebra"
 
 
 class MathNumberTheory(Math):
     VERSION = 1
-    DATASET_NAME = 'number_theory'
+    DATASET_NAME = "number_theory"
 
 
 class MathPrealgebra(Math):
     VERSION = 1
-    DATASET_NAME = 'prealgebra'
+    DATASET_NAME = "prealgebra"
 
 
 class MathPrecalculus(Math):
     VERSION = 1
-    DATASET_NAME = 'precalculus'
+    DATASET_NAME = "precalculus"

lm_eval/tasks/hendrycks_test.py

@@ -3,11 +3,11 @@ Measuring Massive Multitask Language Understanding
 https://arxiv.org/pdf/2009.03300.pdf
 
 The Hendryck's Test is a benchmark that measured a text model’s multitask accuracy.
-The test covers 57 tasks including elementary mathematics, US history, computer 
+The test covers 57 tasks including elementary mathematics, US history, computer
 science, law, and more. To attain high accuracy on this test, models must possess
 extensive world knowledge and problem solving ability. By comprehensively evaluating
-the breadth and depth of a model’s academic and professional understanding, 
-Hendryck's Test can be used to analyze models across many tasks and to identify 
+the breadth and depth of a model’s academic and professional understanding,
+Hendryck's Test can be used to analyze models across many tasks and to identify
 important shortcomings.
 
 Homepage: https://github.com/hendrycks/test
@@ -25,16 +25,65 @@ _CITATION = """
 """
 
 
-SUBJECTS = ['abstract_algebra', 'anatomy', 'astronomy', 'business_ethics', 'clinical_knowledge', 'college_biology',
-            'college_chemistry', 'college_computer_science', 'college_mathematics', 'college_medicine', 'college_physics',
-            'computer_security', 'conceptual_physics', 'econometrics', 'electrical_engineering', 'elementary_mathematics',
-            'formal_logic', 'global_facts', 'high_school_biology', 'high_school_chemistry', 'high_school_computer_science',
-            'high_school_european_history', 'high_school_geography', 'high_school_government_and_politics', 'high_school_macroeconomics',
-            'high_school_mathematics', 'high_school_microeconomics', 'high_school_physics', 'high_school_psychology', 'high_school_statistics',
-            'high_school_us_history', 'high_school_world_history', 'human_aging', 'human_sexuality', 'international_law', 'jurisprudence',
-            'logical_fallacies', 'machine_learning', 'management', 'marketing', 'medical_genetics', 'miscellaneous', 'moral_disputes',
-            'moral_scenarios', 'nutrition', 'philosophy', 'prehistory', 'professional_accounting', 'professional_law', 'professional_medicine',
-            'professional_psychology', 'public_relations', 'security_studies', 'sociology', 'us_foreign_policy', 'virology', 'world_religions']
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
 
 
 def create_all_tasks():
@@ -42,15 +91,14 @@ def create_all_tasks():
     :return: {task_name: task}
         e.g. {hendrycksTest-abstract_algebra: Task, hendrycksTest-anatomy: Task}
     """
-    return {
-        f"hendrycksTest-{sub}": create_task(sub) for sub in SUBJECTS
-    }
+    return {f"hendrycksTest-{sub}": create_task(sub) for sub in SUBJECTS}
 
 
 def create_task(subject):
     class HendrycksTest(GeneralHendrycksTest):
         def __init__(self):
             super().__init__(subject)
+
     return HendrycksTest
 
 
@@ -81,27 +129,32 @@ class GeneralHendrycksTest(MultipleChoiceTask):
     def _process_doc(self, doc):
         def format_example(doc, keys):
             """
-                Question: <prompt>
-                Choices:
-                A. <choice1>
-                B. <choice2>
-                C. <choice3>
-                D. <choice4>
-                Answer:
+            Question: <prompt>
+            Choices:
+            A. <choice1>
+            B. <choice2>
+            C. <choice3>
+            D. <choice4>
+            Answer:
             """
             prompt = "Question: " + doc["question"] + "\nChoices:\n"
-            prompt += "".join([f"{key}. {choice}\n" for key, choice in zip(keys, doc["choices"])])
+            prompt += "".join(
+                [f"{key}. {choice}\n" for key, choice in zip(keys, doc["choices"])]
+            )
             prompt += "Answer:"
             return prompt
-        keys = ['A', 'B', 'C', 'D']
+
+        keys = ["A", "B", "C", "D"]
         return {
             "query": format_example(doc, keys),
             "choices": doc["choices"],
-            "gold": keys.index(doc["answer"]) if isinstance(doc["answer"], str) else doc["answer"]
+            "gold": keys.index(doc["answer"])
+            if isinstance(doc["answer"], str)
+            else doc["answer"],
         }
 
     def fewshot_examples(self, k, rnd):
-        # fewshot_examples is not just sampling from train_docs because dev is 
+        # fewshot_examples is not just sampling from train_docs because dev is
         # in the same distribution as val/test but auxiliary_train isn't
 
         if self._fewshot_docs is None:
@@ -111,3 +164,9 @@ class GeneralHendrycksTest(MultipleChoiceTask):
 
     def doc_to_text(self, doc):
         return doc["query"]
+
+    def should_decontaminate(self):
+        return True
+
+    def doc_to_decontamination_query(self, doc):
+        return doc["query"]

lm_eval/tasks/lambada.py

@@ -20,7 +20,7 @@ from lm_eval.metrics import mean, perplexity
 
 _CITATION = """
 @misc{
-    author={Paperno, Denis and Kruszewski, Germán and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fernández, Raquel}, 
+    author={Paperno, Denis and Kruszewski, Germán and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fernández, Raquel},
     title={The LAMBADA dataset},
     DOI={10.5281/zenodo.2630551},
     publisher={Zenodo},
@@ -53,32 +53,29 @@ class LAMBADA(Task):
         pass
 
     def doc_to_text(self, doc):
-        return doc['text'].rsplit(' ', 1)[0]
+        return doc["text"].rsplit(" ", 1)[0]
+
+    def should_decontaminate(self):
+        return True
+
+    def doc_to_decontamination_query(self, doc):
+        return doc["text"]
 
     def doc_to_target(self, doc):
-        return " " + doc['text'].rsplit(' ', 1)[1]
+        return " " + doc["text"].rsplit(" ", 1)[1]
 
     def construct_requests(self, doc, ctx):
         ll, is_greedy = rf.loglikelihood(ctx, self.doc_to_target(doc))
 
         return ll, is_greedy
-    
+
     def process_results(self, doc, results):
         ll, is_greedy = results
 
-        return {
-            'ppl': ll,
-            'acc': int(is_greedy)
-        }
-        
+        return {"ppl": ll, "acc": int(is_greedy)}
+
     def aggregation(self):
-        return {
-            'ppl': perplexity,
-            'acc': mean
-        }
+        return {"ppl": perplexity, "acc": mean}
 
     def higher_is_better(self):
-        return {
-            'ppl': False,
-            'acc': True
-        }
+        return {"ppl": False, "acc": True}

lm_eval/tasks/lambada_cloze.py

@@ -18,7 +18,7 @@ from lm_eval.tasks.lambada import LAMBADA
 
 _CITATION = """
 @misc{
-    author={Paperno, Denis and Kruszewski, Germán and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fernández, Raquel}, 
+    author={Paperno, Denis and Kruszewski, Germán and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fernández, Raquel},
     title={The LAMBADA dataset},
     DOI={10.5281/zenodo.2630551},
     publisher={Zenodo},
@@ -32,7 +32,13 @@ class LAMBADA_cloze(LAMBADA):
     VERSION = 0
 
     def doc_to_text(self, doc):
-        return doc['text'].rsplit(' ', 1)[0] + " ____. ->"
+        return doc["text"].rsplit(" ", 1)[0] + " ____. ->"
+
+    def should_decontaminate(self):
+        return True
+
+    def doc_to_decontamination_query(self, doc):
+        return doc["text"]
 
     def doc_to_target(self, doc):
-        return " " + doc['text'].rsplit(' ', 1)[1]
+        return " " + doc["text"].rsplit(" ", 1)[1]

lm_eval/tasks/lambada_multilingual.py

@@ -18,7 +18,7 @@ from . import lambada
 
 _CITATION = """
 @misc{
-    author={Paperno, Denis and Kruszewski, Germán and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fernández, Raquel}, 
+    author={Paperno, Denis and Kruszewski, Germán and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fernández, Raquel},
     title={The LAMBADA dataset},
     DOI={10.5281/zenodo.2630551},
     publisher={Zenodo},
@@ -33,28 +33,32 @@ class MultilingualLAMBADA(lambada.LAMBADA):
 
 
 class MultilingualLAMBADAEN(MultilingualLAMBADA):
-    DATASET_NAME = 'en'
+    DATASET_NAME = "en"
 
 
 class MultilingualLAMBADAFR(MultilingualLAMBADA):
-    DATASET_NAME = 'fr'
+    DATASET_NAME = "fr"
 
 
 class MultilingualLAMBADADE(MultilingualLAMBADA):
-    DATASET_NAME = 'de'
+    DATASET_NAME = "de"
 
 
 class MultilingualLAMBADAIT(MultilingualLAMBADA):
-    DATASET_NAME = 'it'
+    DATASET_NAME = "it"
 
 
 class MultilingualLAMBADAES(MultilingualLAMBADA):
-    DATASET_NAME = 'es'
+    DATASET_NAME = "es"
 
 
-LANG_CLASSES = [MultilingualLAMBADAEN, MultilingualLAMBADAFR,
-                MultilingualLAMBADADE, MultilingualLAMBADAIT,
-                MultilingualLAMBADAES]
+LANG_CLASSES = [
+    MultilingualLAMBADAEN,
+    MultilingualLAMBADAFR,
+    MultilingualLAMBADADE,
+    MultilingualLAMBADAIT,
+    MultilingualLAMBADAES,
+]
 
 
 def construct_tasks():

lm_eval/tasks/logiqa.py

@@ -17,7 +17,7 @@ from lm_eval.base import MultipleChoiceTask
 
 _CITATION = """
 @misc{liu2020logiqa,
-    title={LogiQA: A Challenge Dataset for Machine Reading Comprehension with Logical Reasoning}, 
+    title={LogiQA: A Challenge Dataset for Machine Reading Comprehension with Logical Reasoning},
     author={Jian Liu and Leyang Cui and Hanmeng Liu and Dandan Huang and Yile Wang and Yue Zhang},
     year={2020},
     eprint={2007.08124},
@@ -55,14 +55,14 @@ class LogiQA(MultipleChoiceTask):
     def _process_doc(self, doc):
         def format_example(doc, choices):
             """
-                Passage: <passage>
-                Question: <question>
-                Choices:
-                A. <choice1>
-                B. <choice2>
-                C. <choice3>
-                D. <choice4>
-                Answer:
+            Passage: <passage>
+            Question: <question>
+            Choices:
+            A. <choice1>
+            B. <choice2>
+            C. <choice3>
+            D. <choice4>
+            Answer:
             """
             prompt = "Passage: " + doc["context"] + "\n"
             prompt += "Question: " + doc["question"] + "\nChoices:\n"
@@ -70,12 +70,20 @@ class LogiQA(MultipleChoiceTask):
                 prompt += f"{choice.upper()}. {option}\n"
             prompt += "Answer:"
             return prompt
-        choices = ['a', 'b', 'c', 'd']
+
+        choices = ["a", "b", "c", "d"]
         return {
+            "passage": doc["context"],  # Used for decontamination
             "query": format_example(doc, choices),
             "choices": doc["options"],
-            "gold": choices.index(doc["label"])
+            "gold": choices.index(doc["label"]),
         }
 
     def doc_to_text(self, doc):
         return doc["query"]
+
+    def should_decontaminate(self):
+        return True
+
+    def doc_to_decontamination_query(self, doc):
+        return doc["passage"]

lm_eval/tasks/mathqa.py

@@ -14,7 +14,7 @@ from lm_eval.base import MultipleChoiceTask
 
 _CITATION = """
 @misc{amini2019mathqa,
-    title={MathQA: Towards Interpretable Math Word Problem Solving with Operation-Based Formalisms}, 
+    title={MathQA: Towards Interpretable Math Word Problem Solving with Operation-Based Formalisms},
     author={Aida Amini and Saadia Gabriel and Peter Lin and Rik Koncel-Kedziorski and Yejin Choi and Hannaneh Hajishirzi},
     year={2019},
     eprint={1905.13319},
@@ -50,11 +50,14 @@ class MathQA(MultipleChoiceTask):
         return map(self._process_doc, self.dataset["test"])
 
     def _process_doc(self, doc):
-        answer_idx = ['a', 'b', 'c', 'd', 'e'].index(doc['correct'])
-        choices = [c[4:].rstrip(" ,") for c in re.findall(r"[abcd] \) .*?, |e \) .*?$", doc['options'])]
+        answer_idx = ["a", "b", "c", "d", "e"].index(doc["correct"])
+        choices = [
+            c[4:].rstrip(" ,")
+            for c in re.findall(r"[abcd] \) .*?, |e \) .*?$", doc["options"])
+        ]
 
         out_doc = {
-            "query": "Question: " + doc['Problem'] + "\nAnswer:",
+            "query": "Question: " + doc["Problem"] + "\nAnswer:",
             "choices": choices,
             "gold": answer_idx,
         }
@@ -62,3 +65,9 @@ class MathQA(MultipleChoiceTask):
 
     def doc_to_text(self, doc):
         return doc["query"]
+
+    def should_decontaminate(self):
+        return True
+
+    def doc_to_decontamination_query(self, doc):
+        return doc["query"]

lm_eval/tasks/mc_taco.py

@@ -3,18 +3,18 @@
 A Study of Temporal Commonsense Understanding
 https://arxiv.org/pdf/1909.03065.pdf
 
-MC-TACO is a dataset of 13k question-answer pairs that require temporal commonsense 
+MC-TACO is a dataset of 13k question-answer pairs that require temporal commonsense
 comprehension. The dataset contains five temporal properties, (1) duration (how long
-an event takes), (2) temporal ordering (typical order of events), (3) typical time 
+an event takes), (2) temporal ordering (typical order of events), (3) typical time
 (when an event occurs), (4) frequency (how often an event occurs), and (5) stationarity
 (whether a state is maintained for a very long time or indefinitely).
 
-WARNING: Running this task with a `--limit` arg will give misleading results! The 
+WARNING: Running this task with a `--limit` arg will give misleading results! The
 corresponding dataset is structured such that each multiple-choice-question gathered
-by the authors is split into question-option pairs, where each such pair gets 
+by the authors is split into question-option pairs, where each such pair gets
 siloed into an individual document for plausibility testing. Because the harness
 shuffles these documents, setting `--limit` will likely "cut off" certain candidate
-answers. This is a problem because the task's metrics require an exhaustive evaluation 
+answers. This is a problem because the task's metrics require an exhaustive evaluation
 of a question's options. See section 4 of the paper for details.
 
 Homepage: https://leaderboard.allenai.org/mctaco/submissions/public
@@ -55,14 +55,22 @@ class MCTACO(Task):
         return self.dataset["test"]
 
     def doc_to_text(self, doc):
-        return f"{doc['sentence']}\nQuestion: {doc['question']}\n"\
+        return (
+            f"{doc['sentence']}\nQuestion: {doc['question']}\n"
             f"Answer: {doc['answer']}\nPlausible:"
+        )
+
+    def should_decontaminate(self):
+        return True
+
+    def doc_to_decontamination_query(self, doc):
+        return doc["question"] + " " + doc["sentence"]
 
     def doc_to_target(self, doc):
-        return " " + ["no", "yes"][doc['label']]
+        return " " + ["no", "yes"][doc["label"]]
 
     def construct_requests(self, doc, ctx):
-        """ Uses RequestFactory to construct Requests and returns an iterable of
+        """Uses RequestFactory to construct Requests and returns an iterable of
         Requests which will be sent to the LM.
 
         :param doc:
@@ -87,18 +95,15 @@ class MCTACO(Task):
             The results of the requests created in construct_requests.
         """
         ll_no, ll_yes = results
-        gold = doc['label']
+        gold = doc["label"]
         pred = int(ll_yes > ll_no)
         question_id = self._question2id(doc)
         items = (gold, pred, question_id)
-        return {
-            "em": items,
-            "f1": items
-        }
+        return {"em": items, "f1": items}
 
     def _question2id(self, doc):
-        """ Returns an identifier for the question in the given document. """
-        return " ".join([doc['sentence'], doc['question']])
+        """Returns an identifier for the question in the given document."""
+        return " ".join([doc["sentence"], doc["question"]])
 
     def aggregation(self):
         return {
@@ -126,7 +131,7 @@ def exact_match(items):
 
 
 def f1(items):
-    """ See section 4 "Evaluation Metrics" in the paper about the F1 metric used. """
+    """See section 4 "Evaluation Metrics" in the paper about the F1 metric used."""
     results = list(zip(*items))
     # Group the positive ("yes" = 1) golds and predictions by question.
     gold_positives, pred_positives = defaultdict(list), defaultdict(list)
@@ -140,5 +145,5 @@ def f1(items):
         p = tp / pp if pp > 0.0 else 1.0
         r = tp / gp if gp > 0.0 else 1.0
         if p + r > 0.0:
-            f1.append(2. * (p * r) / (p + r))
+            f1.append(2.0 * (p * r) / (p + r))
     return np.mean(f1)

lm_eval/tasks/mutual.py

@@ -29,7 +29,7 @@ class MuTualBase(Task):
     VERSION = 1
     DATASET_PATH = inspect.getfile(lm_eval.datasets.mutual.mutual)
     DATASET_NAME = None
-    CHOICES = ['A', 'B', 'C', 'D']
+    CHOICES = ["A", "B", "C", "D"]
 
     def has_training_docs(self):
         return True
@@ -52,6 +52,12 @@ class MuTualBase(Task):
     def doc_to_text(self, doc):
         return self.detokenize(doc["article"])
 
+    def should_decontaminate(self):
+        return True
+
+    def doc_to_decontamination_query(self, doc):
+        return doc["article"]
+
     def doc_to_target(self, doc):
         return " " + self.detokenize(doc["options"][self.CHOICES.index(doc["answers"])])
 
@@ -82,26 +88,14 @@ class MuTualBase(Task):
         r4_1 = np.argmax(results) == gold  # r4_1 = accuracy
         ranks = sorted(results, reverse=True)
         r4_2 = (ranks.index(results[gold]) == 1) + r4_1
-        mrr = 1. / (ranks.index(results[gold]) + 1)  # `+ 1` for index offset
-        return {
-            "r@1": r4_1,
-            "r@2": r4_2,
-            "mrr": mrr
-        }
+        mrr = 1.0 / (ranks.index(results[gold]) + 1)  # `+ 1` for index offset
+        return {"r@1": r4_1, "r@2": r4_2, "mrr": mrr}
 
     def aggregation(self):
-        return {
-            "r@1": mean,
-            "r@2": mean,
-            "mrr": mean
-        }
+        return {"r@1": mean, "r@2": mean, "mrr": mean}
 
     def higher_is_better(self):
-        return {
-            "r@1": True,
-            "r@2": True,
-            "mrr": True
-        }
+        return {"r@1": True, "r@2": True, "mrr": True}
 
 
 class MuTual(MuTualBase):

lm_eval/tasks/naturalqs.py

-"""
-Natural Questions: a Benchmark for Question Answering Research
-https://storage.googleapis.com/pub-tools-public-publication-data/pdf/1f7b46b5378d757553d3e92ead36bda2e4254244.pdf
-
-The Natural Questions (NQ) corpus is a question-answering dataset that contains
-questions from real users and requires QA systems to read and comprehend an entire
-Wikipedia article that may or may not contain the answer to the question. The
-inclusion of real user questions, and the requirement that solutions should read
-an entire page to find the answer, cause NQ to be a more realistic and challenging
-task than prior QA datasets.
-
-TODO: NaturalQS has a *really* large train set that huggingface just automatically
-downloads even if you dont use it. we should try and only download the val set and
-not even bother with the train set. 
-
-Homepage: https://ai.google.com/research/NaturalQuestions
-"""
-from lm_eval.base import Task
-from itertools import islice
-
-
-_CITATION = """
-@article{47761,
-    title={Natural Questions: a Benchmark for Question Answering Research},
-    author={Tom Kwiatkowski and Jennimaria Palomaki and Olivia Redfield and Michael Collins and Ankur Parikh and Chris Alberti and Danielle Epstein and Illia Polosukhin and Matthew Kelcey and Jacob Devlin and Kenton Lee and Kristina N. Toutanova and Llion Jones and Ming-Wei Chang and Andrew Dai and Jakob Uszkoreit and Quoc Le and Slav Petrov},
-    year={2019},
-    journal={Transactions of the Association of Computational Linguistics}
-}
-"""
-
-
-class NaturalQs(Task):
-    VERSION = 0
-    DATASET_PATH = "natural_questions"
-    DATASET_NAME = None
-
-    def has_training_docs(self):
-        return True
-
-    def has_validation_docs(self):
-        return True
-
-    def has_test_docs(self):
-        return False
-
-    def training_docs(self):
-        # Cache training for faster few-shot.
-        # Data is too large to fit in memory.
-        if self._training_docs is None:
-            self._training_docs = list(self.dataset["train"])
-        return self._training_docs
-
-    def validation_docs(self):
-        return self.dataset["validation"]
-
-    def fewshot_examples(self, k, rnd):
-        # Data is too large to fit in memory. We just sample from the first bit.
-        if self._training_docs is None:
-            self._training_docs = list(islice(self.training_docs(), 0, 100000))
-
-        return rnd.sample(self._training_docs, k)
-
-    def doc_to_text(self, doc):
-        return 'Q: ' + doc['question']['text'] + '\n\n' + 'A:'
-
-    def doc_to_target(self, doc):
-        # There's a short answer and a long answer. Based on the paper, I'm using the long answer.
-        short_answer = doc['annotations']['short_answers'][0]['text']
-        long_answer_start = doc['annotations']['long_answer'][0]['start_token']
-        long_answer_end = doc['annotations']['long_answer'][0]['end_token']
-        long_answer_span = doc['document']['tokens']['token'][long_answer_start:long_answer_end]
-        long_answer_is_html = doc['document']['tokens']['is_html'][long_answer_start:long_answer_end]
-        long_answer_chars = [tok for (tok, is_html) in zip(long_answer_span, long_answer_is_html) if not is_html]
-        long_answer = " ".join(long_answer_chars)
-        return long_answer # Replace with short_answer[0] for short answer
-
-    def construct_requests(self, doc, ctx):
-        """ Uses RequestFactory to construct Requests and returns an iterable of 
-        Requests which will be sent to the LM.
-
-        :param doc:
-            The document as returned from training_docs, validation_docs, or test_docs.
-        :param ctx: str
-            The context string, generated by fewshot_context. This includes the natural 
-            language description, as well as the few shot examples, and the question
-            part of the document for `doc`. 
-        """
-        # TODO: implement evaluation.
-        raise NotImplementedError('Evaluation not implemented')
-    
-    def process_results(self, doc, results):
-        """Take a single document and the LM results and evaluates, returning a 
-        dict where keys are the names of submetrics and values are the values of 
-        the metric for that one document
-
-        :param doc:
-            The document as returned from training_docs, validation_docs, or test_docs.
-        :param results:
-            The results of the requests created in construct_requests.
-        """
-        # TODO: implement evaluation.
-        raise NotImplementedError('Evaluation not implemented')
-
-    def aggregation(self):
-        """
-        :returns: {str: [float] -> float}
-            A dictionary where keys are the names of submetrics and values are 
-            functions that aggregate a list of metrics
-        """
-        # TODO: implement evaluation.
-        raise NotImplementedError('Evaluation not implemented')
-
-    def higher_is_better(self):
-        """
-        :returns: {str: bool}
-            A dictionary where keys are the names of submetrics and values are 
-            whether a higher value of the submetric is better
-        """
-        # TODO: implement evaluation.
-        raise NotImplementedError('Evaluation not implemented')
+"""
+Natural Questions: a Benchmark for Question Answering Research
+https://storage.googleapis.com/pub-tools-public-publication-data/pdf/1f7b46b5378d757553d3e92ead36bda2e4254244.pdf
+
+The Natural Questions (NQ) corpus is a question-answering dataset that contains
+questions from real users and requires QA systems to read and comprehend an entire
+Wikipedia article that may or may not contain the answer to the question. The
+inclusion of real user questions, and the requirement that solutions should read
+an entire page to find the answer, cause NQ to be a more realistic and challenging
+task than prior QA datasets.
+
+TODO: NaturalQS has a *really* large train set that huggingface just automatically
+downloads even if you dont use it. we should try and only download the val set and
+not even bother with the train set.
+
+Homepage: https://ai.google.com/research/NaturalQuestions
+"""
+from lm_eval.base import Task
+from itertools import islice
+
+
+_CITATION = """
+@article{47761,
+    title={Natural Questions: a Benchmark for Question Answering Research},
+    author={Tom Kwiatkowski and Jennimaria Palomaki and Olivia Redfield and Michael Collins and Ankur Parikh and Chris Alberti and Danielle Epstein and Illia Polosukhin and Matthew Kelcey and Jacob Devlin and Kenton Lee and Kristina N. Toutanova and Llion Jones and Ming-Wei Chang and Andrew Dai and Jakob Uszkoreit and Quoc Le and Slav Petrov},
+    year={2019},
+    journal={Transactions of the Association of Computational Linguistics}
+}
+"""
+
+
+class NaturalQs(Task):
+    VERSION = 0
+    DATASET_PATH = "natural_questions"
+    DATASET_NAME = None
+
+    def has_training_docs(self):
+        return True
+
+    def has_validation_docs(self):
+        return True
+
+    def has_test_docs(self):
+        return False
+
+    def training_docs(self):
+        # Cache training for faster few-shot.
+        # Data is too large to fit in memory.
+        if self._training_docs is None:
+            self._training_docs = list(self.dataset["train"])
+        return self._training_docs
+
+    def validation_docs(self):
+        return self.dataset["validation"]
+
+    def fewshot_examples(self, k, rnd):
+        # Data is too large to fit in memory. We just sample from the first bit.
+        if self._training_docs is None:
+            self._training_docs = list(islice(self.training_docs(), 0, 100000))
+
+        return rnd.sample(self._training_docs, k)
+
+    def doc_to_text(self, doc):
+        return "Q: " + doc["question"]["text"] + "\n\n" + "A:"
+
+    def should_decontaminate(self):
+        return True
+
+    def doc_to_decontamination_query(self, doc):
+        return doc["question"]["text"]
+
+    def doc_to_target(self, doc):
+        # There's a short answer and a long answer. Based on the paper, I'm using the long answer.
+        # short_answer = doc["annotations"]["short_answers"][0]["text"]
+        long_answer_start = doc["annotations"]["long_answer"][0]["start_token"]
+        long_answer_end = doc["annotations"]["long_answer"][0]["end_token"]
+        long_answer_span = doc["document"]["tokens"]["token"][
+            long_answer_start:long_answer_end
+        ]
+        long_answer_is_html = doc["document"]["tokens"]["is_html"][
+            long_answer_start:long_answer_end
+        ]
+        long_answer_chars = [
+            tok
+            for (tok, is_html) in zip(long_answer_span, long_answer_is_html)
+            if not is_html
+        ]
+        long_answer = " ".join(long_answer_chars)
+        return long_answer  # Replace with short_answer[0] for short answer
+
+    def construct_requests(self, doc, ctx):
+        """Uses RequestFactory to construct Requests and returns an iterable of
+        Requests which will be sent to the LM.
+
+        :param doc:
+            The document as returned from training_docs, validation_docs, or test_docs.
+        :param ctx: str
+            The context string, generated by fewshot_context. This includes the natural
+            language description, as well as the few shot examples, and the question
+            part of the document for `doc`.
+        """
+        # TODO: implement evaluation.
+        raise NotImplementedError("Evaluation not implemented")
+
+    def process_results(self, doc, results):
+        """Take a single document and the LM results and evaluates, returning a
+        dict where keys are the names of submetrics and values are the values of
+        the metric for that one document
+
+        :param doc:
+            The document as returned from training_docs, validation_docs, or test_docs.
+        :param results:
+            The results of the requests created in construct_requests.
+        """
+        # TODO: implement evaluation.
+        raise NotImplementedError("Evaluation not implemented")
+
+    def aggregation(self):
+        """
+        :returns: {str: [float] -> float}
+            A dictionary where keys are the names of submetrics and values are
+            functions that aggregate a list of metrics
+        """
+        # TODO: implement evaluation.
+        raise NotImplementedError("Evaluation not implemented")
+
+    def higher_is_better(self):
+        """
+        :returns: {str: bool}
+            A dictionary where keys are the names of submetrics and values are
+            whether a higher value of the submetric is better
+        """
+        # TODO: implement evaluation.
+        raise NotImplementedError("Evaluation not implemented")

lm_eval/tasks/openbookqa.py

-"""
-Can a Suit of Armor Conduct Electricity? A New Dataset for Open Book Question Answering
-https://arxiv.org/pdf/1809.02789.pdf
-
-OpenBookQA is a question-answering dataset modeled after open book exams for
-assessing human understanding of a subject. It consists of 5,957 multiple-choice
-elementary-level science questions (4,957 train, 500 dev, 500 test), which probe
-the understanding of a small “book” of 1,326 core science facts and the application
-of these facts to novel situations. For training, the dataset includes a mapping
-from each question to the core science fact it was designed to probe. Answering
-OpenBookQA questions requires additional broad common knowledge, not contained
-in the book. The questions, by design, are answered incorrectly by both a retrieval-
-based algorithm and a word co-occurrence algorithm.
-
-Homepage: https://allenai.org/data/open-book-qa
-"""
-from lm_eval.base import MultipleChoiceTask
-
-
-_CITATION = """
-@inproceedings{OpenBookQA2018,
-    title={Can a Suit of Armor Conduct Electricity? A New Dataset for Open Book Question Answering},
-    author={Todor Mihaylov and Peter Clark and Tushar Khot and Ashish Sabharwal},
-    booktitle={EMNLP},
-    year={2018}
-}
-"""
-
-
-class OpenBookQA(MultipleChoiceTask):
-    VERSION = 0
-    DATASET_PATH = "openbookqa"
-    DATASET_NAME = "main"
-
-    def has_training_docs(self):
-        return True
-
-    def has_validation_docs(self):
-        return True
-
-    def has_test_docs(self):
-        return True
-
-    def training_docs(self):
-        if self._training_docs is None:
-            self._training_docs = list(map(self._process_doc, self.dataset["train"]))
-        return self._training_docs
-
-    def validation_docs(self):
-        return map(self._process_doc, self.dataset["validation"])
-
-    def test_docs(self):
-        return map(self._process_doc, self.dataset["test"])
-
-    def _process_doc(self, doc):
-        out_doc = {
-            "id": doc["id"],
-            "query": doc["question_stem"],
-            "choices": doc["choices"]["text"],
-            "gold": ["A", "B", "C", "D"].index(doc["answerKey"].strip()),
-        }
-        return out_doc
-
-    def doc_to_text(self, doc):
-        return doc["query"]
+"""
+Can a Suit of Armor Conduct Electricity? A New Dataset for Open Book Question Answering
+https://arxiv.org/pdf/1809.02789.pdf
+
+OpenBookQA is a question-answering dataset modeled after open book exams for
+assessing human understanding of a subject. It consists of 5,957 multiple-choice
+elementary-level science questions (4,957 train, 500 dev, 500 test), which probe
+the understanding of a small “book” of 1,326 core science facts and the application
+of these facts to novel situations. For training, the dataset includes a mapping
+from each question to the core science fact it was designed to probe. Answering
+OpenBookQA questions requires additional broad common knowledge, not contained
+in the book. The questions, by design, are answered incorrectly by both a retrieval-
+based algorithm and a word co-occurrence algorithm.
+
+Homepage: https://allenai.org/data/open-book-qa
+"""
+from lm_eval.base import MultipleChoiceTask
+
+
+_CITATION = """
+@inproceedings{OpenBookQA2018,
+    title={Can a Suit of Armor Conduct Electricity? A New Dataset for Open Book Question Answering},
+    author={Todor Mihaylov and Peter Clark and Tushar Khot and Ashish Sabharwal},
+    booktitle={EMNLP},
+    year={2018}
+}
+"""
+
+
+class OpenBookQA(MultipleChoiceTask):
+    VERSION = 0
+    DATASET_PATH = "openbookqa"
+    DATASET_NAME = "main"
+
+    def has_training_docs(self):
+        return True
+
+    def has_validation_docs(self):
+        return True
+
+    def has_test_docs(self):
+        return True
+
+    def training_docs(self):
+        if self._training_docs is None:
+            self._training_docs = list(map(self._process_doc, self.dataset["train"]))
+        return self._training_docs
+
+    def validation_docs(self):
+        return map(self._process_doc, self.dataset["validation"])
+
+    def test_docs(self):
+        return map(self._process_doc, self.dataset["test"])
+
+    def _process_doc(self, doc):
+        out_doc = {
+            "id": doc["id"],
+            "query": doc["question_stem"],
+            "choices": doc["choices"]["text"],
+            "gold": ["A", "B", "C", "D"].index(doc["answerKey"].strip()),
+        }
+        return out_doc
+
+    def doc_to_text(self, doc):
+        return doc["query"]
+
+    def should_decontaminate(self):
+        return True
+
+    def doc_to_decontamination_query(self, doc):
+        return doc["query"]

lm_eval/tasks/piqa.py

@@ -5,7 +5,7 @@ https://arxiv.org/pdf/1911.11641.pdf
 Physical Interaction: Question Answering (PIQA) is a physical commonsense
 reasoning and a corresponding benchmark dataset. PIQA was designed to investigate
 the physical knowledge of existing models. To what extent are current approaches
-actually learning about the world? 
+actually learning about the world?
 
 Homepage: https://yonatanbisk.com/piqa/
 """
@@ -58,3 +58,9 @@ class PiQA(MultipleChoiceTask):
 
     def doc_to_text(self, doc):
         return "Question: " + doc["goal"] + "\nAnswer:"
+
+    def should_decontaminate(self):
+        return True
+
+    def doc_to_decontamination_query(self, doc):
+        return doc["goal"]

lm_eval/tasks/prost.py

@@ -52,22 +52,29 @@ class PROST(MultipleChoiceTask):
     def test_docs(self):
         return map(self._process_doc, self.dataset["test"])
 
-    def fewshot_context(self, doc, num_fewshot, provide_description=None, rnd=None, description=None):
-        assert num_fewshot == 0, 'PROST is designed to probe models in a zero-shot fashion only.'
+    def fewshot_context(
+        self, doc, num_fewshot, provide_description=None, rnd=None, description=None
+    ):
+        assert (
+            num_fewshot == 0
+        ), "PROST is designed to probe models in a zero-shot fashion only."
         return super().fewshot_context(
-            doc=doc,
-            num_fewshot=num_fewshot,
-            rnd=rnd,
-            description=description
+            doc=doc, num_fewshot=num_fewshot, rnd=rnd, description=description
         )
 
     def _process_doc(self, doc):
         out_doc = {
             "query": f"{doc['context']}\nQuestion: {doc['ex_question']}\nAnswer:",
-            "choices": [doc['A'], doc['B'], doc['C'], doc['D']],
-            "gold": doc['label'],
+            "choices": [doc["A"], doc["B"], doc["C"], doc["D"]],
+            "gold": doc["label"],
         }
         return out_doc
 
     def doc_to_text(self, doc):
         return doc["query"]
+
+    def should_decontaminate(self):
+        return True
+
+    def doc_to_decontamination_query(self, doc):
+        return doc["query"]

lm_eval/tasks/pubmedqa.py

@@ -3,14 +3,14 @@ PubMedQA: A Dataset for Biomedical Research Question Answering
 https://arxiv.org/pdf/1909.06146.pdf
 
 PubMedQA is a novel biomedical question answering (QA) dataset collected from
-PubMed abstracts. The task of PubMedQA is to answer research questions with 
-yes/no/maybe (e.g.: Do preoperative statins reduce atrial fibrillation after 
-coronary artery bypass grafting?) using the corresponding abstracts. PubMedQA 
-has 1k expert-annotated, 61.2k unlabeled and 211.3k artificially generated QA 
+PubMed abstracts. The task of PubMedQA is to answer research questions with
+yes/no/maybe (e.g.: Do preoperative statins reduce atrial fibrillation after
+coronary artery bypass grafting?) using the corresponding abstracts. PubMedQA
+has 1k expert-annotated, 61.2k unlabeled and 211.3k artificially generated QA
 instances. Each PubMedQA instance is composed of (1) a question which is either
 an existing research article title or derived from one, (2) a context which is
 the corresponding abstract without its conclusion, (3) a long answer, which is
-the conclusion of the abstract and, presumably, answers the research question, 
+the conclusion of the abstract and, presumably, answers the research question,
 and (4) a yes/no/maybe answer which summarizes the conclusion.
 
 Homepage: https://pubmedqa.github.io/
@@ -53,16 +53,20 @@ class Pubmed_QA(Task):
     def doc_to_text(self, doc):
         ctxs = "\n".join(doc["context"]["contexts"])
         return "Abstract: {}\nQuestion: {}\nAnswer:".format(
-            ctxs,
-            doc["question"],
-            doc["final_decision"]
+            ctxs, doc["question"], doc["final_decision"]
         )
 
+    def should_decontaminate(self):
+        return True
+
+    def doc_to_decontamination_query(self, doc):
+        return doc["question"] + " " + "\n".join(doc["context"]["contexts"])
+
     def doc_to_target(self, doc):
         return " {}".format(doc["final_decision"])
 
     def construct_requests(self, doc, ctx):
-        """ Uses RequestFactory to construct Requests and returns
+        """Uses RequestFactory to construct Requests and returns
         an iterable of Requests which will be sent to the LM.
         """
         ll_yes, _ = rf.loglikelihood(ctx, " yes")
@@ -75,15 +79,11 @@ class Pubmed_QA(Task):
         ll_yes, ll_no, ll_maybe = results
         pred = np.argmax(results)
         return {
-            "acc": ["yes", "no", "maybe"][pred] == gold, 
+            "acc": ["yes", "no", "maybe"][pred] == gold,
         }
 
     def aggregation(self):
-        return {
-            "acc" : mean
-        }
+        return {"acc": mean}
 
     def higher_is_better(self):
-        return {
-            "acc" : True
-        }
+        return {"acc": True}

lm_eval/tasks/qa4mre.py

@@ -3,9 +3,9 @@ QA4MRE 2011-2013: Overview of Question Answering for Machine Reading Evaluation
 https://www.cs.cmu.edu/~./hovy/papers/13CLEF-QA4MRE.pdf
 
 The (English only) QA4MRE challenge which was run as a Lab at CLEF 2011-2013.
-The main objective of this exercise is to develop a methodology for evaluating 
-Machine Reading systems through Question Answering and Reading Comprehension 
-Tests. Systems should be able to extract knowledge from large volumes of text 
+The main objective of this exercise is to develop a methodology for evaluating
+Machine Reading systems through Question Answering and Reading Comprehension
+Tests. Systems should be able to extract knowledge from large volumes of text
 and use this knowledge to answer questions. Four different tasks have been
 organized during these years: Main Task, Processing Modality and Negation for
 Machine Reading, Machine Reading of Biomedical Texts about Alzheimer's disease,
@@ -23,7 +23,7 @@ _CITATION = """
     booktitle={CLEF},
     year={2013}
 }
-"""
+"""  # noqa: W605
 
 
 class QA4MRE(MultipleChoiceTask):
@@ -47,7 +47,7 @@ class QA4MRE(MultipleChoiceTask):
     def _process_doc(self, doc):
         choices = doc["answer_options"]["answer_str"]
         out_doc = {
-            "source": doc["document_str"].strip().replace("\'", "'"),
+            "source": doc["document_str"].strip().replace("'", "'"),
             "query": doc["question_str"],
             "choices": choices,
             "gold": int(doc["correct_answer_id"]) - 1,
@@ -57,6 +57,12 @@ class QA4MRE(MultipleChoiceTask):
     def doc_to_text(self, doc):
         return "{}\nQuestion: {}\nAnswer:".format(doc["source"], doc["query"])
 
+    def should_decontaminate(self):
+        return True
+
+    def doc_to_decontamination_query(self, doc):
+        return doc["source"] + " " + doc["query"]
+
 
 class QA4MRE_2011(QA4MRE):
     DATASET_NAME = "2011.main.EN"

lm_eval/tasks/qasper.py

-""" 
+"""
 A Dataset of Information-Seeking Questions and Answers Anchored in Research Papers
 https://arxiv.org/abs/2105.03011