Merge pull request #292 from jon-tow/task-doc-update

Add citations and descriptions to all tasks

lm_eval/tasks/mathqa.py

+"""
+MathQA: Towards Interpretable Math Word Problem Solving with Operation-Based Formalisms
+https://arxiv.org/pdf/1905.13319.pdf
+
+MathQA is a large-scale dataset of 37k English multiple-choice math word problems
+covering multiple math domain categories by modeling operation programs corresponding
+to word problems in the AQuA dataset (Ling et al., 2017).
+
+Homepage: https://math-qa.github.io/math-QA/
+"""
 import re
 from lm_eval.base import MultipleChoiceTask
 from . common import HFTask
 
 
+_CITATION = """
+@misc{amini2019mathqa,
+    title={MathQA: Towards Interpretable Math Word Problem Solving with Operation-Based Formalisms}, 
+    author={Aida Amini and Saadia Gabriel and Peter Lin and Rik Koncel-Kedziorski and Yejin Choi and Hannaneh Hajishirzi},
+    year={2019},
+    eprint={1905.13319},
+    archivePrefix={arXiv},
+    primaryClass={cs.CL}
+}
+"""
+
+
 class MathQA(HFTask, MultipleChoiceTask):
     VERSION = 0
     DATASET_PATH = "math_qa"

lm_eval/tasks/mc_taco.py

@@ -3,6 +3,12 @@
 A Study of Temporal Commonsense Understanding
 https://arxiv.org/pdf/1909.03065.pdf
 
+MC-TACO is a dataset of 13k question-answer pairs that require temporal commonsense 
+comprehension. The dataset contains five temporal properties, (1) duration (how long
+an event takes), (2) temporal ordering (typical order of events), (3) typical time 
+(when an event occurs), (4) frequency (how often an event occurs), and (5) stationarity
+(whether a state is maintained for a very long time or indefinitely).
+
 WARNING: Running this task with a `--limit` arg will give misleading results! The 
 corresponding dataset is structured such that each multiple-choice-question gathered
 by the authors is split into question-option pairs, where each such pair gets 
@@ -11,6 +17,15 @@ shuffles these documents, setting `--limit` will likely "cut off" certain candid
 answers. This is a problem because the task's metrics require an exhaustive evaluation 
 of a question's options. See section 4 of the paper for details.
 
+Homepage: https://leaderboard.allenai.org/mctaco/submissions/public
+"""
+import numpy as np
+from lm_eval.base import rf
+from collections import defaultdict
+from . common import HFTask
+
+
+_CITATION = """
 @inproceedings{ZKNR19,
     author = {Ben Zhou, Daniel Khashabi, Qiang Ning and Dan Roth},
     title = {“Going on a vacation” takes longer than “Going for a walk”: A Study of Temporal Commonsense Understanding },
@@ -19,11 +34,6 @@ of a question's options. See section 4 of the paper for details.
 }
 """
 
-import numpy as np
-from lm_eval.base import rf
-from collections import defaultdict
-from . common import HFTask
-
 
 class MCTACO(HFTask):
     VERSION = 0

lm_eval/tasks/mutual.py

@@ -2,13 +2,10 @@
 MuTual: A Dataset for Multi-Turn Dialogue Reasoning
 https://www.aclweb.org/anthology/2020.acl-main.130/
 
-@inproceedings{mutual,
-    title = "MuTual: A Dataset for Multi-Turn Dialogue Reasoning",
-    author = "Cui, Leyang  and Wu, Yu and Liu, Shujie and Zhang, Yue and Zhou, Ming" ,
-    booktitle = "Proceedings of the 58th Conference of the Association for Computational Linguistics",
-    year = "2020",
-    publisher = "Association for Computational Linguistics",
-}
+MuTual is a retrieval-based dataset for multi-turn dialogue reasoning, which is
+modified from Chinese high school English listening comprehension test data.
+
+Homepage: https://github.com/Nealcly/MuTual
 """
 import json
 import zipfile
@@ -20,6 +17,17 @@ from lm_eval.metrics import mean
 from best_download import download_file
 
 
+_CITATION = """
+@inproceedings{mutual,
+    title = "MuTual: A Dataset for Multi-Turn Dialogue Reasoning",
+    author = "Cui, Leyang  and Wu, Yu and Liu, Shujie and Zhang, Yue and Zhou, Ming" ,
+    booktitle = "Proceedings of the 58th Conference of the Association for Computational Linguistics",
+    year = "2020",
+    publisher = "Association for Computational Linguistics",
+}
+"""
+
+
 class MuTualBase(Task):
     VERSION = 1
     BASE_PATH = Path("data/mutual")

lm_eval/tasks/naturalqs.py

+"""
+Natural Questions: a Benchmark for Question Answering Research
+https://storage.googleapis.com/pub-tools-public-publication-data/pdf/1f7b46b5378d757553d3e92ead36bda2e4254244.pdf
+
+The Natural Questions (NQ) corpus is a question-answering dataset that contains
+questions from real users and requires QA systems to read and comprehend an entire
+Wikipedia article that may or may not contain the answer to the question. The
+inclusion of real user questions, and the requirement that solutions should read
+an entire page to find the answer, cause NQ to be a more realistic and challenging
+task than prior QA datasets.
+
+TODO: NaturalQS has a *really* large train set that huggingface just automatically
+downloads even if you dont use it. we should try and only download the val set and
+not even bother with the train set. 
+
+Homepage: https://ai.google.com/research/NaturalQuestions
+"""
 import random
 from . common import HFTask
 from itertools import islice
 
 
+_CITATION = """
+@article{47761,
+    title={Natural Questions: a Benchmark for Question Answering Research},
+    author={Tom Kwiatkowski and Jennimaria Palomaki and Olivia Redfield and Michael Collins and Ankur Parikh and Chris Alberti and Danielle Epstein and Illia Polosukhin and Matthew Kelcey and Jacob Devlin and Kenton Lee and Kristina N. Toutanova and Llion Jones and Ming-Wei Chang and Andrew Dai and Jakob Uszkoreit and Quoc Le and Slav Petrov},
+    year={2019},
+    journal={Transactions of the Association of Computational Linguistics}
+}
+"""
+
+
 class NaturalQs(HFTask):
     VERSION = 0
-    # TODO: naturalqs has a *really* large train set that huggingface just
-    # automatically downloads even if you dont use it. we should try and only 
-    # download the val set and not even bother with the train set. 
-
     DATASET_PATH = "natural_questions"
     DATASET_NAME = None
 

lm_eval/tasks/openbookqa.py

+"""
+Can a Suit of Armor Conduct Electricity? A New Dataset for Open Book Question Answering
+https://arxiv.org/pdf/1809.02789.pdf
+
+OpenBookQA is a question-answering dataset modeled after open book exams for
+assessing human understanding of a subject. It consists of 5,957 multiple-choice
+elementary-level science questions (4,957 train, 500 dev, 500 test), which probe
+the understanding of a small “book” of 1,326 core science facts and the application
+of these facts to novel situations. For training, the dataset includes a mapping
+from each question to the core science fact it was designed to probe. Answering
+OpenBookQA questions requires additional broad common knowledge, not contained
+in the book. The questions, by design, are answered incorrectly by both a retrieval-
+based algorithm and a word co-occurrence algorithm.
+
+Homepage: https://allenai.org/data/open-book-qa
+"""
 from lm_eval.base import MultipleChoiceTask
 from .common import HFTask
 
 
+_CITATION = """
+@inproceedings{OpenBookQA2018,
+    title={Can a Suit of Armor Conduct Electricity? A New Dataset for Open Book Question Answering},
+    author={Todor Mihaylov and Peter Clark and Tushar Khot and Ashish Sabharwal},
+    booktitle={EMNLP},
+    year={2018}
+}
+"""
+
+
 class OpenBookQA(HFTask, MultipleChoiceTask):
     VERSION = 0
     DATASET_PATH = "openbookqa"

lm_eval/tasks/pile.py

+"""
+The Pile: An 800GB Dataset of Diverse Text for Language Modeling
+https://arxiv.org/pdf/2101.00027.pdf
+
+The Pile is a 825 GiB diverse, open source language modelling data set that consists
+of 22 smaller, high-quality datasets combined together. To score well on Pile
+BPB (bits per byte), a model must be able to understand many disparate domains
+including books, github repositories, webpages, chat logs, and medical, physics,
+math, computer science, and philosophy papers.
+
+Homepage: https://pile.eleuther.ai/
+"""
 import os
 
 import lm_dataformat
@@ -9,6 +21,16 @@ from ..utils import general_detokenize
 from best_download import download_file
 
 
+_CITATION = """
+@article{pile,
+  title={The {P}ile: An 800GB Dataset of Diverse Text for Language Modeling},
+  author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and Presser, Shawn and Leahy, Connor},
+  journal={arXiv preprint arXiv:2101.00027},
+  year={2020}
+}
+"""
+
+
 class PilePerplexityTask(PerplexityTask, abc.ABC):
     VERSION = 1
 

lm_eval/tasks/piqa.py

+"""
+PIQA: Reasoning about Physical Commonsense in Natural Language
+https://arxiv.org/pdf/1911.11641.pdf
+
+Physical Interaction: Question Answering (PIQA) is a physical commonsense
+reasoning and a corresponding benchmark dataset. PIQA was designed to investigate
+the physical knowledge of existing models. To what extent are current approaches
+actually learning about the world? 
+
+Homepage: https://yonatanbisk.com/piqa/
+"""
 import numpy as np
 from lm_eval.base import MultipleChoiceTask, rf
 from ..metrics import mean
 from . common import HFTask
 
 
+_CITATION = """
+@inproceedings{Bisk2020,
+    author = {Yonatan Bisk and Rowan Zellers and
+            Ronan Le Bras and Jianfeng Gao
+            and Yejin Choi},
+    title = {PIQA: Reasoning about Physical Commonsense in
+           Natural Language},
+    booktitle = {Thirty-Fourth AAAI Conference on
+               Artificial Intelligence},
+    year = {2020},
+}
+"""
+
+
 class PiQA(HFTask, MultipleChoiceTask):
     VERSION = 0
     DATASET_PATH = "piqa"

lm_eval/tasks/prost.py

@@ -2,26 +2,40 @@
 PROST: Physical Reasoning about Objects Through Space and Time
 https://arxiv.org/pdf/2106.03634.pdf
 
+PROST, Physical Reasoning about Objects Through Space and Time, is a dataset
+consisting of 18,736 multiple-choice questions made from 14 manually curated
+templates, covering 10 physical reasoning concepts. All questions are designed
+to probe both causal and masked language models in a zero-shot setting.
+
 NOTE: PROST is limited to the zero-shot setting to adhere to authors' intentions
 as discussed in section 7 of the paper: "We hope that the community will use
 this dataset in the intended way: in a zero-shot setting to probe models which
 have been trained on data not specifically collected to succeed on PROST."
 
-# TODO: Update citation when it is made available at https://github.com/nala-cub/prost.
-@misc{arocaouellette2021prost,
-      title={PROST: Physical Reasoning of Objects through Space and Time}, 
-      author={Stéphane Aroca-Ouellette and Cory Paik and Alessandro Roncone and Katharina Kann},
-      year={2021},
-      eprint={2106.03634},
-      archivePrefix={arXiv},
-      primaryClass={cs.CL}
-}
+Homepage: https://github.com/nala-cub/prost
 """
-
 from lm_eval.base import MultipleChoiceTask
 from . common import HFTask
 
 
+_CITATION = """
+@inproceedings{aroca-ouellette-etal-2021-prost,
+    title = "{PROST}: {P}hysical Reasoning about Objects through Space and Time",
+    author = "Aroca-Ouellette, St{\'e}phane  and
+      Paik, Cory  and
+      Roncone, Alessandro  and
+      Kann, Katharina",
+    booktitle = "Findings of the Association for Computational Linguistics: ACL-IJCNLP 2021",
+    month = aug,
+    year = "2021",
+    address = "Online",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/2021.findings-acl.404",
+    pages = "4597--4608",
+}
+"""
+
+
 class PROST(HFTask, MultipleChoiceTask):
     VERSION = 0
     DATASET_PATH = "corypaik/prost"

lm_eval/tasks/pubmedqa.py

+"""
+PubMedQA: A Dataset for Biomedical Research Question Answering
+https://arxiv.org/pdf/1909.06146.pdf
+
+PubMedQA is a novel biomedical question answering (QA) dataset collected from
+PubMed abstracts. The task of PubMedQA is to answer research questions with 
+yes/no/maybe (e.g.: Do preoperative statins reduce atrial fibrillation after 
+coronary artery bypass grafting?) using the corresponding abstracts. PubMedQA 
+has 1k expert-annotated, 61.2k unlabeled and 211.3k artificially generated QA 
+instances. Each PubMedQA instance is composed of (1) a question which is either
+an existing research article title or derived from one, (2) a context which is
+the corresponding abstract without its conclusion, (3) a long answer, which is
+the conclusion of the abstract and, presumably, answers the research question, 
+and (4) a yes/no/maybe answer which summarizes the conclusion.
+
+Homepage: https://pubmedqa.github.io/
+"""
 import numpy as np
 from .common import HFTask
 from lm_eval.base import rf
 from ..metrics import mean
 
 
+_CITATION = """
+@inproceedings{jin2019pubmedqa,
+    title={PubMedQA: A Dataset for Biomedical Research Question Answering},
+    author={Jin, Qiao and Dhingra, Bhuwan and Liu, Zhengping and Cohen, William and Lu, Xinghua},
+    booktitle={Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)},
+    pages={2567--2577},
+    year={2019}
+}
+"""
+
+
 class Pubmed_QA(HFTask):
     VERSION = 0
     DATASET_PATH = "pubmed_qa"

lm_eval/tasks/qa4mre.py

+"""
+QA4MRE 2011-2013: Overview of Question Answering for Machine Reading Evaluation
+https://www.cs.cmu.edu/~./hovy/papers/13CLEF-QA4MRE.pdf
+
+The (English only) QA4MRE challenge which was run as a Lab at CLEF 2011-2013.
+The main objective of this exercise is to develop a methodology for evaluating 
+Machine Reading systems through Question Answering and Reading Comprehension 
+Tests. Systems should be able to extract knowledge from large volumes of text 
+and use this knowledge to answer questions. Four different tasks have been
+organized during these years: Main Task, Processing Modality and Negation for
+Machine Reading, Machine Reading of Biomedical Texts about Alzheimer's disease,
+and Entrance Exam.
+
+Homepage: http://nlp.uned.es/clef-qa/repository/qa4mre.php
+"""
 import os
 import xml.etree.ElementTree as ET
 from best_download import download_file
 from lm_eval.base import MultipleChoiceTask
 
 
+_CITATION = """
+@inproceedings{Peas2013QA4MRE2O,
+    title={QA4MRE 2011-2013: Overview of Question Answering for Machine Reading Evaluation},
+    author={Anselmo Pe{\~n}as and Eduard H. Hovy and Pamela Forner and {\'A}lvaro Rodrigo and Richard F. E. Sutcliffe and Roser Morante},
+    booktitle={CLEF},
+    year={2013}
+}
+"""
+
+
 class QA4MRE(MultipleChoiceTask):
     VERSION = 0
     YEAR = None

lm_eval/tasks/qasper.py

@@ -2,25 +2,13 @@
 A Dataset of Information-Seeking Questions and Answers Anchored in Research Papers
 https://arxiv.org/abs/2105.03011
 
-@article{DBLP:journals/corr/abs-2105-03011,
-  author    = {Pradeep Dasigi and
-               Kyle Lo and
-               Iz Beltagy and
-               Arman Cohan and
-               Noah A. Smith and
-               Matt Gardner},
-  title     = {A Dataset of Information-Seeking Questions and Answers Anchored in
-               Research Papers},
-  journal   = {CoRR},
-  volume    = {abs/2105.03011},
-  year      = {2021},
-  url       = {https://arxiv.org/abs/2105.03011},
-  eprinttype = {arXiv},
-  eprint    = {2105.03011},
-  timestamp = {Fri, 14 May 2021 12:13:30 +0200},
-  biburl    = {https://dblp.org/rec/journals/corr/abs-2105-03011.bib},
-  bibsource = {dblp computer science bibliography, https://dblp.org}
-}
+QASPER is a dataset of 5,049 questions over 1,585 Natural Language Processing papers.
+Each question is written by an NLP practitioner who read only the title and abstract
+of the corresponding paper, and the question seeks information present in the full
+text. The questions are then answered by a separate set of NLP practitioners who also
+provide supporting evidence to answers.
+
+Homepage: https://allenai.org/data/qasper
 """
 from collections import Counter
 from math import exp
@@ -32,6 +20,29 @@ from lm_eval.metrics import f1_score, mean
 from .common import HFTask
 
 
+_CITATION = """
+@article{DBLP:journals/corr/abs-2105-03011,
+    author    = {Pradeep Dasigi and
+               Kyle Lo and
+               Iz Beltagy and
+               Arman Cohan and
+               Noah A. Smith and
+               Matt Gardner},
+    title     = {A Dataset of Information-Seeking Questions and Answers Anchored in
+               Research Papers},
+    journal   = {CoRR},
+    volume    = {abs/2105.03011},
+    year      = {2021},
+    url       = {https://arxiv.org/abs/2105.03011},
+    eprinttype = {arXiv},
+    eprint    = {2105.03011},
+    timestamp = {Fri, 14 May 2021 12:13:30 +0200},
+    biburl    = {https://dblp.org/rec/journals/corr/abs-2105-03011.bib},
+    bibsource = {dblp computer science bibliography, https://dblp.org}
+}
+"""
+
+
 def normalize_answer(s):
     """
     Taken from the official evaluation script for v1.1 of the SQuAD dataset.

lm_eval/tasks/quac.py

@@ -2,20 +2,30 @@
 QuAC: Question Answering in Context
 https://arxiv.org/abs/1808.07036 
 
-@article{choi2018quac,
-  title={Quac: Question answering in context},
-  author={Choi, Eunsol and He, He and Iyyer, Mohit and Yatskar, Mark and Yih, Wen-tau and Choi, Yejin and Liang, Percy and Zettlemoyer, Luke},
-  journal={arXiv preprint arXiv:1808.07036},
-  year={2018}
-}
-"""
+Question Answering in Context (QuAC) is a dataset for modeling, understanding, and 
+participating in information seeking dialog. Data instances consist of an interactive
+dialog between two crowd workers: (1) a student who poses a sequence of freeform
+questions to learn as much as possible about a hidden Wikipedia text, and (2)
+a teacher who answers the questions by providing short excerpts (spans) from the text.
 
+Homepage: https://quac.ai/
+"""
 import json
 import os
 from lm_eval.base import Task
 from ..utils import sh
 
 
+_CITATION = """
+@article{choi2018quac,
+    title={Quac: Question answering in context},
+    author={Choi, Eunsol and He, He and Iyyer, Mohit and Yatskar, Mark and Yih, Wen-tau and Choi, Yejin and Liang, Percy and Zettlemoyer, Luke},
+    journal={arXiv preprint arXiv:1808.07036},
+    year={2018}
+}
+"""
+
+
 class QuAC(Task):
     VERSION = 0
 

lm_eval/tasks/race.py

+"""
+RACE: Large-scale ReAding Comprehension Dataset From Examinations
+https://arxiv.org/pdf/1704.04683.pdf
+
+RACE is a large-scale reading comprehension dataset with more than 28,000 passages
+and nearly 100,000 questions. The dataset is collected from English examinations
+in China, which are designed for middle school and high school students. The dataset
+can be served as the training and test sets for machine comprehension.
+
+Homepage: https://www.cs.cmu.edu/~glai1/data/race/
+"""
 import collections
 import datasets
 import numpy as np
@@ -6,6 +17,16 @@ from ..metrics import mean
 from . common import HFTask
 
 
+_CITATION = """
+@article{lai2017large,
+    title={RACE: Large-scale ReAding Comprehension Dataset From Examinations},
+    author={Lai, Guokun and Xie, Qizhe and Liu, Hanxiao and Yang, Yiming and Hovy, Eduard},
+    journal={arXiv preprint arXiv:1704.04683},  
+    year={2017}
+}
+"""
+
+
 class each:
     def __init__(self, f):
         self.f = f

lm_eval/tasks/sat.py

+"""
+Similarity of Semantic Relations
+https://arxiv.org/pdf/cs/0608100.pdf
+
+SAT (Scholastic Aptitude Test) Analogy Questions is a dataset comprising 374
+multiple-choice analogy questions; 5 choices per question.
+
+Homepage: https://aclweb.org/aclwiki/SAT_Analogy_Questions_(State_of_the_art)
+"""
 import os
 from lm_eval.base import MultipleChoiceTask
 
 
+_CITATION = """
+@article{article,
+    author = {Turney, Peter},
+    year = {2006},
+    month = {09},
+    pages = {379-416},
+    title = {Similarity of Semantic Relations},
+    volume = {32},
+    journal = {Computational Linguistics},
+    doi = {10.1162/coli.2006.32.3.379}
+}
+"""
+
+
 class SATAnalogies(MultipleChoiceTask):    
     VERSION = 0
     NEEDS_MANUAL_DL = True

lm_eval/tasks/sciq.py

+"""
+Crowdsourcing Multiple Choice Science Questions
+https://aclanthology.org/W17-4413.pdf
+
+The SciQ dataset contains 13,679 crowdsourced science exam questions about Physics,
+Chemistry and Biology, among others. The questions are in multiple-choice format
+with 4 answer options each. For the majority of the questions, an additional paragraph
+with supporting evidence for the correct answer is provided.
+
+Homepage: https://allenai.org/data/sciq
+"""
 import os
 import json
 import zipfile
@@ -5,6 +16,16 @@ from lm_eval.base import MultipleChoiceTask
 from best_download import download_file
 
 
+_CITATION = """
+@inproceedings{Welbl2017CrowdsourcingMC,
+    title={Crowdsourcing Multiple Choice Science Questions},
+    author={Johannes Welbl and Nelson F. Liu and Matt Gardner},
+    booktitle={NUT@EMNLP},
+    year={2017}
+}
+"""
+
+
 class SciQ(MultipleChoiceTask):
     VERSION = 0
     # Multiple languages and multiple years

lm_eval/tasks/squad.py

+"""
+Know What You Don’t Know: Unanswerable Questions for SQuAD
+https://arxiv.org/pdf/1806.03822.pdf
+
+Stanford Question Answering Dataset (SQuAD) is a reading comprehension dataset,
+consisting of questions posed by crowdworkers on a set of Wikipedia articles,
+where the answer to every question is a segment of text, or span, from the
+corresponding reading passage, or the question might be unanswerable.
+SQuAD2.0 combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable
+questions written adversarially by crowdworkers to look similar to answerable ones.
+To do well on SQuAD2.0, systems must not only answer questions when possible, but
+also determine when no answer is supported by the paragraph and abstain from answering.
+
+Homepage: https://rajpurkar.github.io/SQuAD-explorer/
+"""
 import datasets
 from math import exp
 from lm_eval.base import rf
@@ -7,6 +22,18 @@ from functools import partial
 from packaging import version
 
 
+_CITATION = """
+@misc{rajpurkar2018know,
+    title={Know What You Don't Know: Unanswerable Questions for SQuAD}, 
+    author={Pranav Rajpurkar and Robin Jia and Percy Liang},
+    year={2018},
+    eprint={1806.03822},
+    archivePrefix={arXiv},
+    primaryClass={cs.CL}
+}
+"""
+
+
 def _squad_metric(predictions, references):
     squad_metric = datasets.load_metric("squad_v2")
     return squad_metric.compute(predictions=predictions, references=references)

lm_eval/tasks/storycloze.py

+"""
+A Corpus and Cloze Evaluation for Deeper Understanding of Commonsense Stories
+https://arxiv.org/pdf/1604.01696.pdf
+
+'Story Cloze Test' (2018) is a commonsense reasoning framework for evaluating story
+understanding, story generation, and script learning. This test requires a system
+to choose the correct ending to a four-sentence story.
+
+Homepage: https://cs.rochester.edu/nlp/rocstories/
+"""
 import csv
 from lm_eval.base import Task
 
 
+_CITATION = """
+@inproceedings{sharma-etal-2018-tackling,
+    title = "Tackling the Story Ending Biases in The Story Cloze Test",
+    author = "Sharma, Rishi  and
+      Allen, James  and
+      Bakhshandeh, Omid  and
+      Mostafazadeh, Nasrin",
+    booktitle = "Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)",
+    month = jul,
+    year = "2018",
+    address = "Melbourne, Australia",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/P18-2119",
+    doi = "10.18653/v1/P18-2119",
+    pages = "752--757",
+    abstract = "The Story Cloze Test (SCT) is a recent framework for evaluating story comprehension and script learning. There have been a variety of models tackling the SCT so far. Although the original goal behind the SCT was to require systems to perform deep language understanding and commonsense reasoning for successful narrative understanding, some recent models could perform significantly better than the initial baselines by leveraging human-authorship biases discovered in the SCT dataset. In order to shed some light on this issue, we have performed various data analysis and analyzed a variety of top performing models presented for this task. Given the statistics we have aggregated, we have designed a new crowdsourcing scheme that creates a new SCT dataset, which overcomes some of the biases. We benchmark a few models on the new dataset and show that the top-performing model on the original SCT dataset fails to keep up its performance. Our findings further signify the importance of benchmarking NLP systems on various evolving test sets.",
+}
+"""
+
+
 class StoryCloze(Task):
     VERSION = 0
     NEEDS_MANUAL_DL = True

lm_eval/tasks/superglue.py

 """
-To-do:
-    - WSC requires free-form generation
-    - ReCoRD
+SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems
+https://w4ngatang.github.io/static/papers/superglue.pdf
+
+SuperGLUE is a benchmark styled after GLUE with a new set of more difficult language
+understanding tasks.
+
+Homepage: https://super.gluebenchmark.com/
+
+TODO: WSC requires free-form generation.
 """
 import numpy as np
 import sklearn
@@ -12,6 +18,21 @@ from ..metrics import mean, acc_all, metric_max_over_ground_truths
 from ..utils import general_detokenize
 
 
+_CITATION = """
+@inproceedings{NEURIPS2019_4496bf24,
+    author = {Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel},
+    booktitle = {Advances in Neural Information Processing Systems},
+    editor = {H. Wallach and H. Larochelle and A. Beygelzimer and F. d\textquotesingle Alch\'{e}-Buc and E. Fox and R. Garnett},
+    pages = {},
+    publisher = {Curran Associates, Inc.},
+    title = {SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
+    url = {https://proceedings.neurips.cc/paper/2019/file/4496bf24afe7fab6f046bf4923da8de6-Paper.pdf},
+    volume = {32},
+    year = {2019}
+}
+"""
+
+
 class BoolQ(HFTask):
     VERSION = 1
     DATASET_PATH = "super_glue"

lm_eval/tasks/translation.py

+"""
+NOTE: This file implements translation tasks using datasets from WMT conferences,
+provided by sacrebleu. Traditionally they are evaluated with BLEU scores. TER
+and CHRF are other options.
+
+We defer citations and descriptions of the many translations tasks used
+here to the SacreBLEU repo from which we've obtained the datasets:
+https://github.com/mjpost/sacrebleu/blob/master/sacrebleu/dataset.py
+
+Homepage: https://github.com/mjpost/sacrebleu/blob/master/sacrebleu/dataset.py
+"""
 import pycountry
 from pprint import pprint
 from sacrebleu import sacrebleu
@@ -6,13 +17,21 @@ from lm_eval.base import Task, rf
 from typing import List
 
 
-
+_CITATION = """
+@inproceedings{post-2018-call,
+    title = "A Call for Clarity in Reporting {BLEU} Scores",
+    author = "Post, Matt",
+    booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers",
+    month = oct,
+    year = "2018",
+    address = "Belgium, Brussels",
+    publisher = "Association for Computational Linguistics",
+    url = "https://www.aclweb.org/anthology/W18-6319",
+    pages = "186--191",
+}
 """
-This file implements translation tasks using datasets from WMT conferences, provided by sacrebleu.
-Traditionally they are evaluated with BLEU scores. TER and CHRF are other options.
 
-See sacrebleu.DATASETS for all available datasets. There are a lot!
-"""
+
 sacrebleu_datasets = sacrebleu.DATASETS
 
 

lm_eval/tasks/triviaqa.py

+"""
+TriviaQA: A Large Scale Distantly Supervised Challenge Dataset for Reading Comprehension
+https://arxiv.org/pdf/1705.03551.pdf
+
+TriviaQA is a reading comprehension dataset containing over 650K question-answer-evidence
+triples. TriviaQA includes 95K question-answer pairs authored by trivia enthusiasts
+and independently gathered evidence documents, six per question on average, that provide
+high quality distant supervision for answering the questions.
+
+Homepage: https://nlp.cs.washington.edu/triviaqa/
+"""
 import os
 import json
 import jsonlines
@@ -7,6 +18,19 @@ from ..utils import sh
 from best_download import download_file
 
 
+_CITATION = """
+@InProceedings{JoshiTriviaQA2017,
+    author = {Joshi, Mandar and Choi, Eunsol and Weld, Daniel S. and Zettlemoyer, Luke},
+    title = {TriviaQA: A Large Scale Distantly Supervised Challenge Dataset for Reading Comprehension},
+    booktitle = {Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics},
+    month = {July},
+    year = {2017},
+    address = {Vancouver, Canada},
+    publisher = {Association for Computational Linguistics},
+}
+"""
+
+
 class TriviaQA(Task):
     VERSION = 0
     def download(self):