Add citations and descriptions to all tasks

docs/task_guide.md

@@ -23,13 +23,36 @@ cd lm_eval/tasks
 touch <task-name>.py
 ```
 
-Then open the file and create a multiline docstring on the first line with the name of the paper associated with your task/s on one line, the paper’s url on the next line, and its BibTeX Code on another. For example, take the QuAC dataset. You’d write:
+Then open the file and create a multiline docstring on the first line with the following contents:
+
+```python
+"""
+<Paper title>
+<Paper PDF URL>
+
+<Short description of task>
+
+Homepage: <URL to task's homepage>
+
+<Citation>
+"""
+```
+
+For example, take the QuAC dataset. We have:
 
 ```python
 """
 QuAC: Question Answering in Context
 https://arxiv.org/abs/1808.07036
 
+Question Answering in Context (QuAC) is a dataset for modeling, understanding, and 
+participating in information seeking dialog. Data instances consist of an interactive
+dialog between two crowd workers: (1) a student who poses a sequence of freeform
+questions to learn as much as possible about a hidden Wikipedia text, and (2)
+a teacher who answers the questions by providing short excerpts (spans) from the text.
+
+Homepage: https://quac.ai/
+
 @article{choi2018quac,
   title={Quac: Question answering in context},
   author={Choi, Eunsol and He, He and Iyyer, Mohit and Yatskar, Mark and Yih, Wen-tau and Choi, Yejin and Liang, Percy and Zettlemoyer, Luke},

lm_eval/tasks/anli.py

+"""
+Adversarial NLI: A New Benchmark for Natural Language Understanding
+https://arxiv.org/pdf/1910.14599.pdf
+
+Adversarial NLI (ANLI) is a dataset collected via an iterative, adversarial
+human-and-model-in-the-loop procedure. It consists of three rounds that progressively
+increase in difficulty and complexity, and each question-answer includes annotator-
+provided explanations.
+
+Homepage: "https://github.com/facebookresearch/anli"
+
+@inproceedings{nie-etal-2020-adversarial,
+    title = "Adversarial {NLI}: A New Benchmark for Natural Language Understanding",
+    author = "Nie, Yixin  and
+      Williams, Adina  and
+      Dinan, Emily  and
+      Bansal, Mohit  and
+      Weston, Jason  and
+      Kiela, Douwe",
+    booktitle = "Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics",
+    year = "2020",
+    publisher = "Association for Computational Linguistics",
+}
+"""
 import numpy as np
 from lm_eval.base import rf
 from ..metrics import mean

lm_eval/tasks/arc.py

+"""
+Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge
+https://arxiv.org/pdf/1803.05457.pdf
+
+The ARC dataset consists of 7,787 science exam questions drawn from a variety
+of sources, including science questions provided under license by a research
+partner affiliated with AI2. These are text-only, English language exam questions
+that span several grade levels as indicated in the files. Each question has a
+multiple choice structure (typically 4 answer options). The questions are sorted
+into a Challenge Set of 2,590 “hard” questions (those that both a retrieval and
+a co-occurrence method fail to answer correctly) and an Easy Set of 5,197 questions.
+
+Homepage: https://allenai.org/data/arc
+
+@article{Clark2018ThinkYH,
+  title={Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge},
+  author={Peter Clark and Isaac Cowhey and Oren Etzioni and Tushar Khot and Ashish Sabharwal and Carissa Schoenick and Oyvind Tafjord},
+  journal={ArXiv},
+  year={2018},
+  volume={abs/1803.05457}
+}
+"""
 from lm_eval.base import MultipleChoiceTask
 from . common import HFTask
 

lm_eval/tasks/arithmetic.py

+"""
+Language Models are Few-Shot Learners
+https://arxiv.org/pdf/2005.14165.pdf
+
+A small battery of 10 tests that involve asking language models a simple arithmetic
+problem in natural language.
+
+Homepage: https://github.com/openai/gpt-3/tree/master/data
+
+@inproceedings{NEURIPS2020_1457c0d6,
+ author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},
+ booktitle = {Advances in Neural Information Processing Systems},
+ editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},
+ pages = {1877--1901},
+ publisher = {Curran Associates, Inc.},
+ title = {Language Models are Few-Shot Learners},
+ url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},
+ volume = {33},
+ year = {2020}
+}
+"""
 import abc
 import json
 import os

lm_eval/tasks/asdiv.py

@@ -2,6 +2,18 @@
 ASDiv: A Diverse Corpus for Evaluating and Developing English Math Word Problem Solvers
 https://arxiv.org/abs/2106.15772
 
+ASDiv (Academia Sinica Diverse MWP Dataset) is a diverse (in terms of both language
+patterns and problem types) English math word problem (MWP) corpus for evaluating
+the capability of various MWP solvers. Existing MWP corpora for studying AI progress
+remain limited either in language usage patterns or in problem types. We thus present
+a new English MWP corpus with 2,305 MWPs that cover more text patterns and most problem
+types taught in elementary school. Each MWP is annotated with its problem type and grade
+level (for indicating the level of difficulty).
+
+NOTE: We currently ignore formulas for answer generation.
+
+Homepage: https://github.com/chaochun/nlu-asdiv-dataset
+
 @misc{miao2021diverse,
       title={A Diverse Corpus for Evaluating and Developing English Math Word Problem Solvers},
       author={Shen-Yun Miao and Chao-Chun Liang and Keh-Yih Su},
@@ -21,9 +33,7 @@ import numpy as np
 from zipfile import ZipFile
 import os 
 
-#currently ignoring formula for answer generation
 
-# given a subset, splits return the docs 
 class Asdiv(Task):
     VERSION = 0
     DATASET_PATH = Path("data/asdiv")

lm_eval/tasks/blimp.py

@@ -2,11 +2,26 @@
 BLiMP: A Benchmark of Linguistic Minimal Pairs for English
 https://arxiv.org/abs/1912.00582
 
+BLiMP is a challenge set for evaluating what language models (LMs) know about
+major grammatical phenomena in English. BLiMP consists of 67 sub-datasets, each
+containing 1000 minimal pairs isolating specific contrasts in syntax, morphology,
+or semantics. The data is automatically generated according to expert-crafted
+grammars.
+
+Homepage: https://github.com/alexwarstadt/blimp
+
 @article{warstadt2019blimp,
-  title={BLiMP: A Benchmark of Linguistic Minimal Pairs for English},
-  author={Warstadt, Alex and Parrish, Alicia and Liu, Haokun and Mohananey, Anhad and Peng, Wei, and Wang, Sheng-Fu and Bowman, Samuel R},
-  journal={arXiv preprint arXiv:1912.00582},
-  year={2019}
+    author = {Warstadt, Alex and Parrish, Alicia and Liu, Haokun and Mohananey, Anhad and Peng, Wei and Wang, Sheng-Fu and Bowman, Samuel R.},
+    title = {BLiMP: The Benchmark of Linguistic Minimal Pairs for English},
+    journal = {Transactions of the Association for Computational Linguistics},
+    volume = {8},
+    number = {},
+    pages = {377-392},
+    year = {2020},
+    doi = {10.1162/tacl\_a\_00321},
+    URL = {https://doi.org/10.1162/tacl_a_00321},
+    eprint = {https://doi.org/10.1162/tacl_a_00321},
+    abstract = { We introduce The Benchmark of Linguistic Minimal Pairs (BLiMP),1 a challenge set for evaluating the linguistic knowledge of language models (LMs) on major grammatical phenomena in English. BLiMP consists of 67 individual datasets, each containing 1,000 minimal pairs—that is, pairs of minimally different sentences that contrast in grammatical acceptability and isolate specific phenomenon in syntax, morphology, or semantics. We generate the data according to linguist-crafted grammar templates, and human aggregate agreement with the labels is 96.4\%. We evaluate n-gram, LSTM, and Transformer (GPT-2 and Transformer-XL) LMs by observing whether they assign a higher probability to the acceptable sentence in each minimal pair. We find that state-of-the-art models identify morphological contrasts related to agreement reliably, but they struggle with some subtle semantic and syntactic phenomena, such as negative polarity items and extraction islands. }
 }
 """
 

lm_eval/tasks/cbt.py

+"""
+The Children’s Book Test (CBT) from the paper:
+https://research.fb.com/wp-content/uploads/2016/11/the_goldilocks_principle_reading_children_s_books_with_explicit_memory_representations.pdf
+
+The Children's Book Test (CBT) is test of how well language models capture 
+meaning in children's books. Unlike standard language modelling benchmarks,
+it distinguishes the task of predicting syntactic function words from that
+of predicting lower-frequency words, which carry greater semantic content.
+
+NOTE: This evaluation is based on the (context + query) question-answering variant
+used by the Recurrent Language Models described in the paper. See section 4.4.
+
+Homepage: https://github.com/facebookresearch/ParlAI/tree/main/parlai/tasks/cbt
+    
+@misc{hill2016goldilocks,
+      title={The Goldilocks Principle: Reading Children's Books with Explicit Memory Representations}, 
+      author={Felix Hill and Antoine Bordes and Sumit Chopra and Jason Weston},
+      year={2016},
+      eprint={1511.02301},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+"""
 import numpy as np
 from lm_eval.base import rf
 from lm_eval.metrics import mean
@@ -5,17 +28,10 @@ from .common import HFTask
 
 
 class CBTBase(HFTask):
-    """The Children’s Book Test (CBT) from the paper:
-    https://research.fb.com/wp-content/uploads/2016/11/the_goldilocks_principle_reading_children_s_books_with_explicit_memory_representations.pdf
-    NOTE: This evaluation is based on the (context + query) question-answering variant
-    used by the Recurrent Language Models described in the aforementioned paper.
-    See section 4.4.
-    """
-
+    VERSION = 0
     DATASET_PATH = "cbt"
     DATASET_NAME = None
 
-    VERSION = 0
 
     def detokenize(self, text):
         text = text.replace(" '", "'")

lm_eval/tasks/coqa.py

+"""
+CoQA: A Conversational Question Answering Challenge
+https://arxiv.org/pdf/1808.07042.pdf
+
+CoQA is a large-scale dataset for building Conversational Question Answering 
+systems. The goal of the CoQA challenge is to measure the ability of machines to 
+understand a text passage and answer a series of interconnected questions that 
+appear in a conversation.
+
+Homepage: https://stanfordnlp.github.io/coqa/
+
+@misc{reddy2018coqa,
+    title={CoQA: A Conversational Question Answering Challenge},
+    author={Siva Reddy and Danqi Chen and Christopher D. Manning},
+    year={2018},
+    eprint={1808.07042},
+    archivePrefix={arXiv},
+    primaryClass={cs.CL}
+}
+"""
 import os
 import json
 import transformers.data.metrics.squad_metrics as squad_metrics

lm_eval/tasks/drop.py

+"""
+DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs
+https://aclanthology.org/attachments/N19-1246.Supplementary.pdf
+
+DROP is a QA dataset which tests comprehensive understanding of paragraphs. In 
+this crowdsourced, adversarially-created, 96k question-answering benchmark, a 
+system must resolve multiple references in a question, map them onto a paragraph,
+and perform discrete operations over them (such as addition, counting, or sorting).
+
+Homepage: https://allenai.org/data/drop
+
+Acknowledgement: This implementation is based on the official evaluation for `DROP`:
+https://github.com/allenai/allennlp-reading-comprehension/blob/master/allennlp_rc/eval/drop_eval.py
+
+@misc{dua2019drop,
+      title={DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs}, 
+      author={Dheeru Dua and Yizhong Wang and Pradeep Dasigi and Gabriel Stanovsky and Sameer Singh and Matt Gardner},
+      year={2019},
+      eprint={1903.00161},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+"""
 import json
 import numpy as np
 import re
@@ -9,13 +32,9 @@ from lm_eval.metrics import mean
 from pathlib import Path
 from zipfile import ZipFile
 
-"""
-Acknowledgement: This implementation is based on the official evaluation for `DROP`:
-https://github.com/allenai/allennlp-reading-comprehension/blob/master/allennlp_rc/eval/drop_eval.py
-"""
-
 _ARTICLES = re.compile(r"\b(a|an|the)\b", re.UNICODE)
 
+
 class DROP(Task):
     VERSION = 1
     DATASET_PATH = Path("data/drop")

lm_eval/tasks/glue.py

+"""
+GLUE: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding
+https://openreview.net/pdf?id=rJ4km2R5t7
+
+The General Language Understanding Evaluation (GLUE) benchmark is a collection of
+resources for training, evaluating, and analyzing natural language understanding
+systems. GLUE consists of:
+- A benchmark of nine sentence- or sentence-pair language understanding tasks built
+on established existing datasets and selected to cover a diverse range of dataset
+sizes, text genres, and degrees of difficulty, and
+- A diagnostic dataset designed to evaluate and analyze model performance with
+respect to a wide range of linguistic phenomena found in natural language.
+
+Homepage: https://gluebenchmark.com/
+
+@inproceedings{wang-etal-2018-glue,
+    title = "{GLUE}: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding",
+    author = "Wang, Alex  and
+      Singh, Amanpreet  and
+      Michael, Julian  and
+      Hill, Felix  and
+      Levy, Omer  and
+      Bowman, Samuel",
+    booktitle = "Proceedings of the 2018 {EMNLP} Workshop {B}lackbox{NLP}: Analyzing and Interpreting Neural Networks for {NLP}",
+    month = nov,
+    year = "2018",
+    address = "Brussels, Belgium",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/W18-5446",
+    doi = "10.18653/v1/W18-5446",
+    pages = "353--355",
+    abstract = "Human ability to understand language is \textit{general, flexible, and robust}. In contrast, most NLU models above the word level are designed for a specific task and struggle with out-of-domain data. If we aspire to develop models with understanding beyond the detection of superficial correspondences between inputs and outputs, then it is critical to develop a unified model that can execute a range of linguistic tasks across different domains. To facilitate research in this direction, we present the General Language Understanding Evaluation (GLUE, gluebenchmark.com): a benchmark of nine diverse NLU tasks, an auxiliary dataset for probing models for understanding of specific linguistic phenomena, and an online platform for evaluating and comparing models. For some benchmark tasks, training data is plentiful, but for others it is limited or does not match the genre of the test set. GLUE thus favors models that can represent linguistic knowledge in a way that facilitates sample-efficient learning and effective knowledge-transfer across tasks. While none of the datasets in GLUE were created from scratch for the benchmark, four of them feature privately-held test data, which is used to ensure that the benchmark is used fairly. We evaluate baselines that use ELMo (Peters et al., 2018), a powerful transfer learning technique, as well as state-of-the-art sentence representation models. The best models still achieve fairly low absolute scores. Analysis with our diagnostic dataset yields similarly weak performance over all phenomena tested, with some exceptions.",
+}
+"""
 import numpy as np
 from lm_eval.base import rf
 from ..metrics import mean, matthews_corrcoef, f1_score

lm_eval/tasks/gsm8k.py

@@ -2,6 +2,20 @@
 "Training Verifiers to Solve Math Word Problems"
 https://arxiv.org/abs/2110.14168
 
+State-of-the-art language models can match human performance on many tasks, but 
+they still struggle to robustly perform multi-step mathematical reasoning. To 
+diagnose the failures of current models and support research, we introduce GSM8K,
+a dataset of 8.5K high quality linguistically diverse grade school math word problems.
+We find that even the largest transformer models fail to achieve high test performance, 
+despite the conceptual simplicity of this problem distribution.
+
+NOTE: See the official implementation of the task: 
+    https://github.com/openai/grade-school-math/blob/master/grade_school_math/calculator.py
+for how to make use of the dataset's calculator annotations in your language
+model's sample/generation function.
+
+Homepage: https://github.com/openai/grade-school-math
+
 @misc{cobbe2021training,
       title={Training Verifiers to Solve Math Word Problems},
       author={Karl Cobbe and Vineet Kosaraju and Mohammad Bavarian and Jacob Hilton and Reiichiro Nakano and Christopher Hesse and John Schulman},
@@ -10,11 +24,6 @@ https://arxiv.org/abs/2110.14168
       archivePrefix={arXiv},
       primaryClass={cs.LG}
 }
-
-NOTE: See the official implementation of the task: 
-    https://github.com/openai/grade-school-math/blob/master/grade_school_math/calculator.py
-for how to make use of the dataset's calculator annotations in your language
-model's sample/generation function.
 """
 
 import json

lm_eval/tasks/headqa.py

+"""
+Interpretable Multi-Step Reasoning with Knowledge Extraction on Complex Healthcare Question Answering
+https://aclanthology.org/P19-1092.pdf
+
+HEAD-QA is a multi-choice HEAlthcare Dataset. The questions come from exams to 
+access a specialized position in the Spanish healthcare system, and are challenging
+even for highly specialized humans.
+
+Homepage: https://aghie.github.io/head-qa/
+
+@misc{liu2020interpretable,
+      title={Interpretable Multi-Step Reasoning with Knowledge Extraction on Complex Healthcare Question Answering}, 
+      author={Ye Liu and Shaika Chowdhury and Chenwei Zhang and Cornelia Caragea and Philip S. Yu},
+      year={2020},
+      eprint={2008.02434},
+      archivePrefix={arXiv},
+      primaryClass={cs.AI}
+}
+"""
 from . common import HFTask
 from lm_eval.base import MultipleChoiceTask
 

lm_eval/tasks/hellaswag.py

+"""
+HellaSwag: Can a Machine Really Finish Your Sentence?
+https://arxiv.org/pdf/1905.07830.pdf
+
+Hellaswag is a commonsense inference challenge dataset. Though its questions are
+trivial for humans (>95% accuracy), state-of-the-art models struggle (<48%). This is
+achieved via Adversarial Filtering (AF), a data collection paradigm wherein a
+series of discriminators iteratively select an adversarial set of machine-generated
+wrong answers. AF proves to be surprisingly robust. The key insight is to scale up
+the length and complexity of the dataset examples towards a critical 'Goldilocks'
+zone wherein generated text is ridiculous to humans, yet often misclassified by
+state-of-the-art models.
+
+Homepage: https://rowanzellers.com/hellaswag/
+
+@inproceedings{zellers2019hellaswag,
+    title={HellaSwag: Can a Machine Really Finish Your Sentence?},
+    author={Zellers, Rowan and Holtzman, Ari and Bisk, Yonatan and Farhadi, Ali and Choi, Yejin},
+    booktitle ={Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics},
+    year={2019}
+}
+"""
 import re
 from lm_eval.base import MultipleChoiceTask
 from . common import HFTask

lm_eval/tasks/hendrycks_ethics.py

+"""
+Aligning AI With Shared Human Values
+https://arxiv.org/pdf/2008.02275.pdf
+
+The ETHICS dataset is a benchmark that spans concepts in justice, well-being,
+duties, virtues, and commonsense morality. Models predict widespread moral
+judgments about diverse text scenarios. This requires connecting physical and
+social world knowledge to value judgements, a capability that may enable us
+to steer chatbot outputs or eventually regularize open-ended reinforcement
+learning agents.
+
+NOTE: The reported "group" accuracies for the Deontology, Justice, and Virtue
+tasks are refered to in this work as the `em` sub-metric. See Section 3. Metrics.
+of the paper.
+
+Homepage: https://github.com/hendrycks/ethics
+
+@article{hendrycks2021ethics,
+  title={Aligning AI With Shared Human Values},
+  author={Dan Hendrycks and Collin Burns and Steven Basart and Andrew Critch and Jerry Li and Dawn Song and Jacob Steinhardt},
+  journal={Proceedings of the International Conference on Learning Representations (ICLR)},
+  year={2021}
+}
+""" 
 import abc
 import csv
 import os
@@ -9,12 +33,6 @@ from lm_eval.utils import sh
 from .common import yesno
 from best_download import download_file
 
-"""
-NOTE: The reported "group" accuracies for the Deontology, Justice, and Virtue
-tasks are refered to in this work as the `em` sub-metric. See Section 3. Metrics.
-of the paper.
-"""
-
 
 class Ethics(Task):
     def download(self):

lm_eval/tasks/hendrycks_math.py

+"""
+Measuring Mathematical Problem Solving With the MATH Dataset
+https://arxiv.org/pdf/2103.03874.pdf
+
+Math is a dataset of 12,500 challenging competition mathematics problems. Each
+problem in Math has a full step-by-step solution which can be used to teach
+models to generate answer derivations and explanations.
+
+Homepage: https://github.com/hendrycks/math
+
+@article{hendrycksmath2021,
+  title={Measuring Mathematical Problem Solving With the Math Dataset},
+  author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt},
+  journal={NeurIPS},
+  year={2021}
+}
+"""
 import abc
 import json
 from lm_eval.utils import sh
@@ -8,11 +25,6 @@ from best_download import download_file
 
 
 class Math(Task):
-    """
-    This dataset is based on the following paper:
-    https://arxiv.org/abs/2103.03874 
-    """
-
     DATASET_PATH = Path('data/MATH')
 
     def download(self):

lm_eval/tasks/hendrycks_test.py

+"""
+Measuring Massive Multitask Language Understanding
+https://arxiv.org/pdf/2009.03300.pdf
+
+The Hendryck's Test is a benchmark that measured a text model’s multitask accuracy.
+The test covers 57 tasks including elementary mathematics, US history, computer 
+science, law, and more. To attain high accuracy on this test, models must possess
+extensive world knowledge and problem solving ability. By comprehensively evaluating
+the breadth and depth of a model’s academic and professional understanding, 
+Hendryck's Test can be used to analyze models across many tasks and to identify 
+important shortcomings.
+
+Homepage: https://github.com/hendrycks/test
+
+@article{hendryckstest2021,
+  title={Measuring Massive Multitask Language Understanding},
+  author={Dan Hendrycks and Collin Burns and Steven Basart and Andy Zou and Mantas Mazeika and Dawn Song and Jacob Steinhardt},
+  journal={Proceedings of the International Conference on Learning Representations (ICLR)},
+  year={2021}
+}
+"""
 import csv
 import random
 from lm_eval.base import MultipleChoiceTask

lm_eval/tasks/lambada.py

+"""
+The LAMBADA dataset: Word prediction requiring a broad discourse context∗
+https://arxiv.org/pdf/1606.06031.pdf
+
+LAMBADA is a dataset to evaluate the capabilities of computational models for text
+understanding by means of a word prediction task. LAMBADA is a collection of narrative
+passages sharing the characteristic that human subjects are able to guess their last
+word if they are exposed to the whole passage, but not if they only see the last
+sentence preceding the target word. To succeed on LAMBADA, computational models
+cannot simply rely on local context, but must be able to keep track of information
+in the broader discourse.
+
+Homepage: https://zenodo.org/record/2630551#.X4Xzn5NKjUI
+
+@misc{
+  author={Paperno, Denis and Kruszewski, Germán and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fernández, Raquel}, 
+  title={The LAMBADA dataset},
+  DOI={10.5281/zenodo.2630551},
+  publisher={Zenodo},
+  year={2016},
+  month={Aug}
+}
+"""
 import json
 from lm_eval.base import Task, rf
 from lm_eval.metrics import mean, perplexity

lm_eval/tasks/lambada_multilingual.py

+"""
+The LAMBADA dataset: Word prediction requiring a broad discourse context∗
+https://arxiv.org/pdf/1606.06031.pdf
+
+The LAMBADA dataset machine-translated to other languages.
+LAMBADA is a dataset to evaluate the capabilities of computational models for text
+understanding by means of a word prediction task. LAMBADA is a collection of narrative
+passages sharing the characteristic that human subjects are able to guess their last
+word if they are exposed to the whole passage, but not if they only see the last
+sentence preceding the target word. To succeed on LAMBADA, computational models
+cannot simply rely on local context, but must be able to keep track of information
+in the broader discourse.
+
+Homepage: https://zenodo.org/record/2630551#.X4Xzn5NKjUI
+
+@misc{
+  author={Paperno, Denis and Kruszewski, Germán and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fernández, Raquel}, 
+  title={The LAMBADA dataset},
+  DOI={10.5281/zenodo.2630551},
+  publisher={Zenodo},
+  year={2016},
+  month={Aug}
+}
+"""
 from . import lambada
 from lm_eval.base import Task, rf
 from lm_eval.metrics import mean, perplexity
@@ -7,7 +31,6 @@ import json
 from functools import partial
 import os 
 
-# This task is lambada but machine-translated to the other languages.
 
 LANGS = ["en", "fr", "de", "it", "es"]
 CHECKSUMS = {"en": "4aa8d02cd17c719165fc8a7887fddd641f43fcafa4b1c806ca8abc31fabdb226", 

lm_eval/tasks/logiqa.py

+"""
+LogiQA: A Challenge Dataset for Machine Reading Comprehension with Logical Reasoning
+https://arxiv.org/pdf/2007.08124.pdf
+
+LogiQA is a dataset for testing human logical reasoning. It consists of 8,678 QA
+instances, covering multiple types of deductive reasoning. Results show that state-
+of-the-art neural models perform by far worse than human ceiling. The dataset can
+also serve as a benchmark for reinvestigating logical AI under the deep learning
+NLP setting.
+
+Homepage: https://github.com/lgw863/LogiQA-dataset
+
+@misc{liu2020logiqa,
+      title={LogiQA: A Challenge Dataset for Machine Reading Comprehension with Logical Reasoning}, 
+      author={Jian Liu and Leyang Cui and Hanmeng Liu and Dandan Huang and Yile Wang and Yue Zhang},
+      year={2020},
+      eprint={2007.08124},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}"""
 from lm_eval.base import MultipleChoiceTask
 from best_download import download_file
 from pathlib import Path

lm_eval/tasks/mathqa.py

+"""
+MathQA: Towards Interpretable Math Word Problem Solving with Operation-Based Formalisms
+https://arxiv.org/pdf/1905.13319.pdf
+
+MathQA is a large-scale dataset of 37k English multiple-choice math word problems
+covering multiple math domain categories by modeling operation programs corresponding
+to word problems in the AQuA dataset (Ling et al., 2017).
+
+Homepage: https://math-qa.github.io/math-QA/
+
+@misc{amini2019mathqa,
+      title={MathQA: Towards Interpretable Math Word Problem Solving with Operation-Based Formalisms}, 
+      author={Aida Amini and Saadia Gabriel and Peter Lin and Rik Koncel-Kedziorski and Yejin Choi and Hannaneh Hajishirzi},
+      year={2019},
+      eprint={1905.13319},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+"""
 import re
 from lm_eval.base import MultipleChoiceTask
 from . common import HFTask