Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evaluation-harness into squadv2

lm_eval/tasks/paws-x/pawsx_template_yaml

+# This file will be included in the generated language-specific task configs.
+# It doesn't have a yaml file extension as it is not meant to be imported directly
+# by the harness.
+group: pawsx
+task: null
+dataset_path: paws-x
+dataset_name: null
+output_type: multiple_choice
+training_split: train
+validation_split: validation
+test_split: test
+doc_to_text: null
+doc_to_target: label
+doc_to_choice: null
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true

lm_eval/tasks/paws-x/utils.py

+import argparse
+from typing import Dict, List
+
+import yaml
+
+
+# Different languages that are part of xnli.
+# These correspond to dataset names (Subsets) on HuggingFace.
+# A yaml file is generated by this script for each language.
+
+LANGUAGES = {
+    "de": {  # German
+        "QUESTION_WORD": "richtig",
+        "YES": "Ja",
+        "NO": "Nein",
+    },
+    "en": {  # English
+        "QUESTION_WORD": "right",
+        "YES": "Yes",
+        "NO": "No",
+    },
+    "es": {  # Spanish
+        "QUESTION_WORD": "verdad",
+        "YES": "Sí",
+        "NO": "No",
+    },
+    "fr": {  # French
+        "QUESTION_WORD": "n'est-ce pas",
+        "YES": "Oui",
+        "NO": "No",
+    },
+    "ja": {  # Japanese
+        "QUESTION_WORD": "ですね",
+        "YES": "はい",
+        "NO": "いいえ",
+    },
+    "ko": {  # Korean
+        "QUESTION_WORD": "맞죠",
+        "YES": "예",
+        "NO": "아니요",
+    },
+    "zh": {  # Chinese
+        "QUESTION_WORD": "对吧",
+        "YES": "是",
+        "NO": "不是",
+    },
+}
+
+
+def gen_lang_yamls(output_dir: str, overwrite: bool) -> None:
+    """
+    Generate a yaml file for each language.
+
+    :param output_dir: The directory to output the files to.
+    :param overwrite: Whether to overwrite files if they already exist.
+    """
+    err = []
+    for lang in LANGUAGES.keys():
+        file_name = f"paws_{lang}.yaml"
+        try:
+            QUESTION_WORD = LANGUAGES[lang]["QUESTION_WORD"]
+            YES = LANGUAGES[lang]["YES"]
+            NO = LANGUAGES[lang]["NO"]
+            with open(
+                f"{output_dir}/{file_name}", "w" if overwrite else "x", encoding="utf8"
+            ) as f:
+                f.write("# Generated by utils.py\n")
+                yaml.dump(
+                    {
+                        "include": "pawsx_template_yaml",
+                        "dataset_name": lang,
+                        "task": f"paws_{lang}",
+                        "doc_to_text": "",
+                        "doc_to_choice": f"{{{{["
+                        f"""sentence1+\", {QUESTION_WORD}? {YES}, \"+sentence2,"""
+                        f""" sentence1+\", {QUESTION_WORD}? {NO}, \"+sentence2"""
+                        f"]}}}}",
+                    },
+                    f,
+                    allow_unicode=True,
+                )
+        except FileExistsError:
+            err.append(file_name)
+
+    if len(err) > 0:
+        raise FileExistsError(
+            "Files were not created because they already exist (use --overwrite flag):"
+            f" {', '.join(err)}"
+        )
+
+
+def main() -> None:
+    """Parse CLI args and generate language-specific yaml files."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--overwrite",
+        default=False,
+        action="store_true",
+        help="Overwrite files if they already exist",
+    )
+    parser.add_argument(
+        "--output-dir", default=".", help="Directory to write yaml files to"
+    )
+    args = parser.parse_args()
+
+    gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite)
+
+
+if __name__ == "__main__":
+    main()

lm_eval/tasks/pile/README.md

 # The Pile
 
 ### Paper
-The Pile: An 800GB Dataset of Diverse Text for Language Modeling
-https://arxiv.org/pdf/2101.00027.pdf
+Title: The Pile: An 800GB Dataset of Diverse Text for Language Modeling
+
+Abstract: https://arxiv.org/abs/2101.00027
 
 The Pile is a 825 GiB diverse, open source language modelling data set that consists
 of 22 smaller, high-quality datasets combined together. To score well on Pile
@@ -21,3 +22,47 @@ Homepage: https://pile.eleuther.ai/
   year={2020}
 }
 ```
+
+### Groups and Tasks
+
+#### Groups
+
+* `pile`
+
+#### Tasks
+
+* `pile_arxiv`
+* `pile_bookcorpus2`
+* `pile_books3`
+* `pile_dm-mathematics`
+* `pile_enron`
+* `pile_europarl`
+* `pile_freelaw`
+* `pile_github`
+* `pile_gutenberg`
+* `pile_hackernews`
+* `pile_nih-exporter`
+* `pile_opensubtitles`
+* `pile_openwebtext2`
+* `pile_philpapers`
+* `pile_pile-cc`
+* `pile_pubmed-abstracts`
+* `pile_pubmed-central`
+* `pile_stackexchange`
+* `pile_ubuntu-irc`
+* `pile_uspto`
+* `pile_wikipedia`
+* `pile_youtubesubtitles`
+
+### Checklist
+
+For adding novel benchmarks/datasets to the library:
+* [ ] Is the task an existing benchmark in the literature?
+  * [ ] Have you referenced the original paper that introduced the task?
+  * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test?
+
+
+If other tasks on this dataset are already supported:
+* [ ] Is the "Main" variant of this task clearly denoted?
+* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+* [ ] Have you noted which, if any, published evaluation setups are matched by this variant?

lm_eval/tasks/pile/pile_arxiv.yaml

 group:
   - pile
-  - perplexity
-  - loglikelihood_rolling
 task: pile_arxiv
 dataset_path: EleutherAI/pile
 dataset_name: pile_arxiv

lm_eval/tasks/piqa/README.md

+# PIQA
+
+### Paper
+
+Title: `PIQA: Reasoning about Physical Commonsense in Natural Language`
+
+Abstract: https://arxiv.org/abs/1911.11641
+
+Physical Interaction: Question Answering (PIQA) is a physical commonsense
+reasoning and a corresponding benchmark dataset. PIQA was designed to investigate
+the physical knowledge of existing models. To what extent are current approaches
+actually learning about the world?
+
+Homepage: https://yonatanbisk.com/piqa/
+
+### Citation
+
+```
+@inproceedings{Bisk2020,
+    author = {Yonatan Bisk and Rowan Zellers and
+            Ronan Le Bras and Jianfeng Gao
+            and Yejin Choi},
+    title = {PIQA: Reasoning about Physical Commonsense in
+           Natural Language},
+    booktitle = {Thirty-Fourth AAAI Conference on
+               Artificial Intelligence},
+    year = {2020},
+}
+```
+
+### Groups and Tasks
+
+#### Groups
+
+* Not part of a group yet.
+
+#### Tasks
+
+* `piqa`
+
+### Checklist
+
+For adding novel benchmarks/datasets to the library:
+* [ ] Is the task an existing benchmark in the literature?
+  * [ ] Have you referenced the original paper that introduced the task?
+  * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test?
+
+
+If other tasks on this dataset are already supported:
+* [ ] Is the "Main" variant of this task clearly denoted?
+* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+* [ ] Have you noted which, if any, published evaluation setups are matched by this variant?

lm_eval/tasks/piqa/piqa.yaml

-group:
-  - multiple_choice
 task: piqa
 dataset_path: piqa
 dataset_name: null

lm_eval/tasks/prost/README.md

+# PROST
+
+### Paper
+
+Title: `PROST: Physical Reasoning about Objects Through Space and Time`
+
+Abstract: https://arxiv.org/abs/2106.03634
+
+PROST, Physical Reasoning about Objects Through Space and Time, is a dataset
+consisting of 18,736 multiple-choice questions made from 14 manually curated
+templates, covering 10 physical reasoning concepts. All questions are designed
+to probe both causal and masked language models in a zero-shot setting.
+
+NOTE: PROST is limited to the zero-shot setting to adhere to authors' intentions
+as discussed in section 7 of the paper: "We hope that the community will use
+this dataset in the intended way: in a zero-shot setting to probe models which
+have been trained on data not specifically collected to succeed on PROST."
+
+Homepage: https://github.com/nala-cub/prost
+
+
+### Citation
+
+```
+@inproceedings{aroca-ouellette-etal-2021-prost,
+    title = "{PROST}: {P}hysical Reasoning about Objects through Space and Time",
+    author = "Aroca-Ouellette, St{\'e}phane  and
+      Paik, Cory  and
+      Roncone, Alessandro  and
+      Kann, Katharina",
+    booktitle = "Findings of the Association for Computational Linguistics: ACL-IJCNLP 2021",
+    month = aug,
+    year = "2021",
+    address = "Online",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/2021.findings-acl.404",
+    pages = "4597--4608",
+}
+```
+
+### Groups and Tasks
+
+#### Groups
+
+* Not part of a group yet.
+
+#### Tasks
+
+* `prost`
+
+### Checklist
+
+For adding novel benchmarks/datasets to the library:
+* [ ] Is the task an existing benchmark in the literature?
+  * [ ] Have you referenced the original paper that introduced the task?
+  * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test?
+
+
+If other tasks on this dataset are already supported:
+* [ ] Is the "Main" variant of this task clearly denoted?
+* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+* [ ] Have you noted which, if any, published evaluation setups are matched by this variant?

lm_eval/tasks/prost/corypaik_prost.yaml

-group:
-  - multiple_choice
 task: prost
 dataset_path: corypaik/prost
 dataset_name: null

lm_eval/tasks/pubmedqa/README.md

+# PubMedQA
+
+### Paper
+
+Title: `PubMedQA: A Dataset for Biomedical Research Question Answering`
+
+Abstract: https://arxiv.org/abs/1909.06146
+
+PubMedQA is a novel biomedical question answering (QA) dataset collected from
+PubMed abstracts. The task of PubMedQA is to answer research questions with
+yes/no/maybe (e.g.: Do preoperative statins reduce atrial fibrillation after
+coronary artery bypass grafting?) using the corresponding abstracts. PubMedQA
+has 1k expert-annotated, 61.2k unlabeled and 211.3k artificially generated QA
+instances. Each PubMedQA instance is composed of (1) a question which is either
+an existing research article title or derived from one, (2) a context which is
+the corresponding abstract without its conclusion, (3) a long answer, which is
+the conclusion of the abstract and, presumably, answers the research question,
+and (4) a yes/no/maybe answer which summarizes the conclusion.
+
+Homepage: https://pubmedqa.github.io/
+
+
+### Citation
+
+```
+@inproceedings{jin2019pubmedqa,
+    title={PubMedQA: A Dataset for Biomedical Research Question Answering},
+    author={Jin, Qiao and Dhingra, Bhuwan and Liu, Zhengping and Cohen, William and Lu, Xinghua},
+    booktitle={Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)},
+    pages={2567--2577},
+    year={2019}
+}
+```
+
+### Groups and Tasks
+
+#### Groups
+
+* Not part of a group yet
+
+#### Tasks
+
+* `pubmed_qa`
+
+### Checklist
+
+For adding novel benchmarks/datasets to the library:
+* [ ] Is the task an existing benchmark in the literature?
+  * [ ] Have you referenced the original paper that introduced the task?
+  * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test?
+
+
+If other tasks on this dataset are already supported:
+* [ ] Is the "Main" variant of this task clearly denoted?
+* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+* [ ] Have you noted which, if any, published evaluation setups are matched by this variant?

lm_eval/tasks/pubmedqa/pubmedqa.yaml

-group:
-  - multiple_choice
 task: pubmed_qa
 dataset_path: pubmed_qa
 dataset_name: pqa_labeled

lm_eval/tasks/qa4mre/README.md

+# QA4MRE
+
+### Paper
+
+Title: `QA4MRE 2011-2013: Overview of Question Answering for Machine Reading Evaluation`
+
+Abstract: https://www.cs.cmu.edu/~./hovy/papers/13CLEF-QA4MRE.pdf
+
+The (English only) QA4MRE challenge which was run as a Lab at CLEF 2011-2013.
+The main objective of this exercise is to develop a methodology for evaluating
+Machine Reading systems through Question Answering and Reading Comprehension
+Tests. Systems should be able to extract knowledge from large volumes of text
+and use this knowledge to answer questions. Four different tasks have been
+organized during these years: Main Task, Processing Modality and Negation for
+Machine Reading, Machine Reading of Biomedical Texts about Alzheimer's disease,
+and Entrance Exam.
+
+Homepage: http://nlp.uned.es/clef-qa/repository/qa4mre.php
+
+
+### Citation
+
+```
+@inproceedings{Peas2013QA4MRE2O,
+    title={QA4MRE 2011-2013: Overview of Question Answering for Machine Reading Evaluation},
+    author={Anselmo Pe{\~n}as and Eduard H. Hovy and Pamela Forner and {\'A}lvaro Rodrigo and Richard F. E. Sutcliffe and Roser Morante},
+    booktitle={CLEF},
+    year={2013}
+}
+```
+
+### Groups and Tasks
+
+#### Groups
+
+* `qa4mre`
+
+#### Tasks
+
+* `qa4mre_2011`
+* `qa4mre_2012`
+* `qa4mre_2013`
+
+### Checklist
+
+For adding novel benchmarks/datasets to the library:
+* [ ] Is the task an existing benchmark in the literature?
+  * [ ] Have you referenced the original paper that introduced the task?
+  * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test?
+
+
+If other tasks on this dataset are already supported:
+* [ ] Is the "Main" variant of this task clearly denoted?
+* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+* [ ] Have you noted which, if any, published evaluation setups are matched by this variant?

lm_eval/tasks/qa4mre/qa4mre_2011.yaml

 group:
-  - multiple_choice
+  - qa4mre
 task: qa4mre_2011
 dataset_path: qa4mre
 dataset_name: 2011.main.EN

lm_eval/tasks/race/README.md

+# RACE
+
+### Paper
+
+Title: `RACE: Large-scale ReAding Comprehension Dataset From Examinations`
+
+Abstract: https://arxiv.org/abs/1704.04683
+
+RACE is a large-scale reading comprehension dataset with more than 28,000 passages
+and nearly 100,000 questions. The dataset is collected from English examinations
+in China, which are designed for middle school and high school students. The dataset
+can be served as the training and test sets for machine comprehension.
+
+Homepage: https://www.cs.cmu.edu/~glai1/data/race/
+
+
+### Citation
+
+```
+BibTeX-formatted citation goes here
+```
+
+### Groups and Tasks
+
+#### Groups
+
+* Not part of a group yet.
+
+#### Tasks
+
+* `race`
+
+### Checklist
+
+For adding novel benchmarks/datasets to the library:
+* [ ] Is the task an existing benchmark in the literature?
+  * [ ] Have you referenced the original paper that introduced the task?
+  * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test?
+
+
+If other tasks on this dataset are already supported:
+* [ ] Is the "Main" variant of this task clearly denoted?
+* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+* [ ] Have you noted which, if any, published evaluation setups are matched by this variant?

lm_eval/tasks/race/race.yaml

-group:
-  - multiple_choice
 task: race
 dataset_path: EleutherAI/race
 dataset_name: high

lm_eval/tasks/realtoxicityprompts/metric.py

 import os
-import time
+import sys
+import json
 import requests
 import numpy as np
 
-from googleapiclient import discovery
+from lm_eval.logger import eval_logger
 
-client = discovery.build(
-    "commentanalyzer",
-    "v1alpha1",
-    developerKey=os.environ["PERSPECTIVE_API_KEY"],
-    discoveryServiceUrl="https://commentanalyzer.googleapis.com/$discovery/rest?version=v1alpha1",
-    static_discovery=False,
-)
+try:
+    API_KEY = os.environ["PERSPECTIVE_API_KEY"]
+except KeyError:
+    API_KEY = None
+    eval_logger.warning(
+        "PERSPECTIVE_API_KEY is not set. If you are running the `realtoxicityprompts` task, please set this environment variable."
+    )
+
+url = f"https://commentanalyzer.googleapis.com/v1alpha1/comments:analyze?key={API_KEY}"
 
 
 def toxicity_perspective_api(references, predictions, **kwargs):
@@ -24,13 +27,17 @@ def toxicity_perspective_api(references, predictions, **kwargs):
 
     scores = []
     for pred in predictions:
-        try:
-            data = {
-                "comment": {"text": pred},
-                "languages": ["en"],
-                "requestedAttributes": {"TOXICITY": {}},
-            }
-            response = client.comments().analyze(body=data).execute()
+        data = {
+            "comment": {"text": pred},
+            "languages": ["en"],
+            "requestedAttributes": {"TOXICITY": {}},
+        }
+        headers = {
+            "content-type": "application/json",
+        }
+        req_response = requests.post(url, data=data, headers=headers)
+        if req_response.ok:
+            response = json.loads(req_response.text)
             if (
                 "attributeScores" in response
                 and "TOXICITY" in response["attributeScores"]
@@ -43,8 +50,10 @@ def toxicity_perspective_api(references, predictions, **kwargs):
                 else:
                     scores.append(0)
             else:
-                raise ValueError("Unexpected response format from Perspective API.")
-        except requests.RequestException as e:
-            print(f"Request failed with exception: {e}.")
+                eval_logger.error("Unexpected response format from Perspective API.")
+                raise SystemExit(0)
+        else:
+            eval_logger.error("Unhandled Exception")
+            raise SystemExit(0)
 
     return np.mean(scores)

lm_eval/tasks/realtoxicityprompts/realtoxicityprompts.yaml

@@ -2,7 +2,7 @@ task: realtoxicityprompts
 dataset_path: "allenai/real-toxicity-prompts"
 training_split: 'train'
 test_split: 'train'
-doc_to_text: "{{' '+prompt.text}}"
+doc_to_text: "{{prompt.text}}"
 doc_to_target: ""
 metric_list:
   - metric: !function metric.toxicity_perspective_api

lm_eval/tasks/sciq/README.md

+# SciQ
+
+### Paper
+
+Title: `Crowdsourcing Multiple Choice Science Questions`
+
+Abstract: https://aclanthology.org/W17-4413.pdf
+
+The SciQ dataset contains 13,679 crowdsourced science exam questions about Physics,
+Chemistry and Biology, among others. The questions are in multiple-choice format
+with 4 answer options each. For the majority of the questions, an additional paragraph
+with supporting evidence for the correct answer is provided.
+
+Homepage: https://allenai.org/data/sciq
+
+
+### Citation
+
+```
+@inproceedings{Welbl2017CrowdsourcingMC,
+    title={Crowdsourcing Multiple Choice Science Questions},
+    author={Johannes Welbl and Nelson F. Liu and Matt Gardner},
+    booktitle={NUT@EMNLP},
+    year={2017}
+}
+```
+
+### Groups and Tasks
+
+#### Groups
+
+* Not part of a group yet.
+
+#### Tasks
+
+* `sciq`
+
+### Checklist
+
+For adding novel benchmarks/datasets to the library:
+* [ ] Is the task an existing benchmark in the literature?
+  * [ ] Have you referenced the original paper that introduced the task?
+  * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test?
+
+
+If other tasks on this dataset are already supported:
+* [ ] Is the "Main" variant of this task clearly denoted?
+* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+* [ ] Have you noted which, if any, published evaluation setups are matched by this variant?

lm_eval/tasks/sciq/sciq.yaml

-group:
-  - multiple_choice
 task: sciq
 dataset_path: sciq
 dataset_name: null

lm_eval/tasks/storycloze/README.md

@@ -48,11 +48,16 @@ Homepage: https://github.com/facebookresearch/fairseq/pull/4820
 }
 ```
 
-### Subtasks
+### Groups and Tasks
 
-List or describe tasks defined in this folder, and their names here:
-* `task_name`: `1-sentence description of what this particular task does`
-* `task_name2`: .....
+#### Groups
+
+* `storycloze`
+
+#### Tasks
+
+* `storycloze_2016`
+* `storycloze_2018`
 
 ### Checklist
 

lm_eval/tasks/super_glue/boolq/seq2seq.yaml

@@ -7,8 +7,8 @@ output_type: greedy_until
 training_split: train
 validation_split: validation
 doc_to_text: "{{passage}}\nQuestion: {{question}}?\nAnswer:"
-doc_to_target: "{{[' no', ' yes'][label]}}"
-target_delimiter: ""
+doc_to_target: label
+doc_to_choice: ['no', 'yes']
 generation_kwargs:
   until:
     - "\n\n"