Make citations module-level constants

lm_eval/tasks/mc_taco.py

@@ -18,7 +18,14 @@ answers. This is a problem because the task's metrics require an exhaustive eval
 of a question's options. See section 4 of the paper for details.
 
 Homepage: https://leaderboard.allenai.org/mctaco/submissions/public
+"""
+import numpy as np
+from lm_eval.base import rf
+from collections import defaultdict
+from . common import HFTask
+
 
+_CITATION = """
 @inproceedings{ZKNR19,
     author = {Ben Zhou, Daniel Khashabi, Qiang Ning and Dan Roth},
     title = {“Going on a vacation” takes longer than “Going for a walk”: A Study of Temporal Commonsense Understanding },
@@ -27,11 +34,6 @@ Homepage: https://leaderboard.allenai.org/mctaco/submissions/public
 }
 """
 
-import numpy as np
-from lm_eval.base import rf
-from collections import defaultdict
-from . common import HFTask
-
 
 class MCTACO(HFTask):
     VERSION = 0

lm_eval/tasks/mutual.py

@@ -6,14 +6,6 @@ MuTual is a retrieval-based dataset for multi-turn dialogue reasoning, which is
 modified from Chinese high school English listening comprehension test data.
 
 Homepage: https://github.com/Nealcly/MuTual
-
-@inproceedings{mutual,
-    title = "MuTual: A Dataset for Multi-Turn Dialogue Reasoning",
-    author = "Cui, Leyang  and Wu, Yu and Liu, Shujie and Zhang, Yue and Zhou, Ming" ,
-    booktitle = "Proceedings of the 58th Conference of the Association for Computational Linguistics",
-    year = "2020",
-    publisher = "Association for Computational Linguistics",
-}
 """
 import json
 import zipfile
@@ -25,6 +17,17 @@ from lm_eval.metrics import mean
 from best_download import download_file
 
 
+_CITATION = """
+@inproceedings{mutual,
+    title = "MuTual: A Dataset for Multi-Turn Dialogue Reasoning",
+    author = "Cui, Leyang  and Wu, Yu and Liu, Shujie and Zhang, Yue and Zhou, Ming" ,
+    booktitle = "Proceedings of the 58th Conference of the Association for Computational Linguistics",
+    year = "2020",
+    publisher = "Association for Computational Linguistics",
+}
+"""
+
+
 class MuTualBase(Task):
     VERSION = 1
     BASE_PATH = Path("data/mutual")

lm_eval/tasks/naturalqs.py

@@ -14,19 +14,22 @@ downloads even if you dont use it. we should try and only download the val set a
 not even bother with the train set. 
 
 Homepage: https://ai.google.com/research/NaturalQuestions
-
-@article{47761,
-title	= {Natural Questions: a Benchmark for Question Answering Research},
-author	= {Tom Kwiatkowski and Jennimaria Palomaki and Olivia Redfield and Michael Collins and Ankur Parikh and Chris Alberti and Danielle Epstein and Illia Polosukhin and Matthew Kelcey and Jacob Devlin and Kenton Lee and Kristina N. Toutanova and Llion Jones and Ming-Wei Chang and Andrew Dai and Jakob Uszkoreit and Quoc Le and Slav Petrov},
-year	= {2019},
-journal	= {Transactions of the Association of Computational Linguistics}
-}
 """
 import random
 from . common import HFTask
 from itertools import islice
 
 
+_CITATION = """
+@article{47761,
+    title={Natural Questions: a Benchmark for Question Answering Research},
+    author={Tom Kwiatkowski and Jennimaria Palomaki and Olivia Redfield and Michael Collins and Ankur Parikh and Chris Alberti and Danielle Epstein and Illia Polosukhin and Matthew Kelcey and Jacob Devlin and Kenton Lee and Kristina N. Toutanova and Llion Jones and Ming-Wei Chang and Andrew Dai and Jakob Uszkoreit and Quoc Le and Slav Petrov},
+    year={2019},
+    journal={Transactions of the Association of Computational Linguistics}
+}
+"""
+
+
 class NaturalQs(HFTask):
     VERSION = 0
     DATASET_PATH = "natural_questions"

lm_eval/tasks/openbookqa.py

@@ -13,7 +13,12 @@ in the book. The questions, by design, are answered incorrectly by both a retrie
 based algorithm and a word co-occurrence algorithm.
 
 Homepage: https://allenai.org/data/open-book-qa
+"""
+from lm_eval.base import MultipleChoiceTask
+from .common import HFTask
 
+
+_CITATION = """
 @inproceedings{OpenBookQA2018,
     title={Can a Suit of Armor Conduct Electricity? A New Dataset for Open Book Question Answering},
     author={Todor Mihaylov and Peter Clark and Tushar Khot and Ashish Sabharwal},
@@ -21,8 +26,6 @@ Homepage: https://allenai.org/data/open-book-qa
     year={2018}
 }
 """
-from lm_eval.base import MultipleChoiceTask
-from .common import HFTask
 
 
 class OpenBookQA(HFTask, MultipleChoiceTask):

lm_eval/tasks/pile.py

@@ -9,13 +9,6 @@ including books, github repositories, webpages, chat logs, and medical, physics,
 math, computer science, and philosophy papers.
 
 Homepage: https://pile.eleuther.ai/
-
-@article{pile,
-  title={The {P}ile: An 800GB Dataset of Diverse Text for Language Modeling},
-  author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and Presser, Shawn and Leahy, Connor},
-  journal={arXiv preprint arXiv:2101.00027},
-  year={2020}
-}
 """
 import os
 
@@ -28,6 +21,16 @@ from ..utils import general_detokenize
 from best_download import download_file
 
 
+_CITATION = """
+@article{pile,
+  title={The {P}ile: An 800GB Dataset of Diverse Text for Language Modeling},
+  author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and Presser, Shawn and Leahy, Connor},
+  journal={arXiv preprint arXiv:2101.00027},
+  year={2020}
+}
+"""
+
+
 class PilePerplexityTask(PerplexityTask, abc.ABC):
     VERSION = 1
 

lm_eval/tasks/piqa.py

@@ -8,7 +8,14 @@ the physical knowledge of existing models. To what extent are current approaches
 actually learning about the world? 
 
 Homepage: https://yonatanbisk.com/piqa/
+"""
+import numpy as np
+from lm_eval.base import MultipleChoiceTask, rf
+from ..metrics import mean
+from . common import HFTask
 
+
+_CITATION = """
 @inproceedings{Bisk2020,
     author = {Yonatan Bisk and Rowan Zellers and
             Ronan Le Bras and Jianfeng Gao
@@ -20,10 +27,6 @@ Homepage: https://yonatanbisk.com/piqa/
     year = {2020},
 }
 """
-import numpy as np
-from lm_eval.base import MultipleChoiceTask, rf
-from ..metrics import mean
-from . common import HFTask
 
 
 class PiQA(HFTask, MultipleChoiceTask):

lm_eval/tasks/prost.py

@@ -13,7 +13,12 @@ this dataset in the intended way: in a zero-shot setting to probe models which
 have been trained on data not specifically collected to succeed on PROST."
 
 Homepage: https://github.com/nala-cub/prost
+"""
+from lm_eval.base import MultipleChoiceTask
+from . common import HFTask
+
 
+_CITATON = """
 @inproceedings{aroca-ouellette-etal-2021-prost,
     title = "{PROST}: {P}hysical Reasoning about Objects through Space and Time",
     author = "Aroca-Ouellette, St{\'e}phane  and
@@ -30,9 +35,6 @@ Homepage: https://github.com/nala-cub/prost
 }
 """
 
-from lm_eval.base import MultipleChoiceTask
-from . common import HFTask
-
 
 class PROST(HFTask, MultipleChoiceTask):
     VERSION = 0

lm_eval/tasks/pubmedqa.py

@@ -14,7 +14,14 @@ the conclusion of the abstract and, presumably, answers the research question,
 and (4) a yes/no/maybe answer which summarizes the conclusion.
 
 Homepage: https://pubmedqa.github.io/
+"""
+import numpy as np
+from .common import HFTask
+from lm_eval.base import rf
+from ..metrics import mean
 
+
+_CITATION = """
 @inproceedings{jin2019pubmedqa,
     title={PubMedQA: A Dataset for Biomedical Research Question Answering},
     author={Jin, Qiao and Dhingra, Bhuwan and Liu, Zhengping and Cohen, William and Lu, Xinghua},
@@ -23,10 +30,6 @@ Homepage: https://pubmedqa.github.io/
     year={2019}
 }
 """
-import numpy as np
-from .common import HFTask
-from lm_eval.base import rf
-from ..metrics import mean
 
 
 class Pubmed_QA(HFTask):

lm_eval/tasks/qa4mre.py

@@ -12,7 +12,14 @@ Machine Reading, Machine Reading of Biomedical Texts about Alzheimer's disease,
 and Entrance Exam.
 
 Homepage: http://nlp.uned.es/clef-qa/repository/qa4mre.php
+"""
+import os
+import xml.etree.ElementTree as ET
+from best_download import download_file
+from lm_eval.base import MultipleChoiceTask
 
+
+_CITATION = """
 @inproceedings{Peas2013QA4MRE2O,
     title={QA4MRE 2011-2013: Overview of Question Answering for Machine Reading Evaluation},
     author={Anselmo Pe{\~n}as and Eduard H. Hovy and Pamela Forner and {\'A}lvaro Rodrigo and Richard F. E. Sutcliffe and Roser Morante},
@@ -20,10 +27,6 @@ Homepage: http://nlp.uned.es/clef-qa/repository/qa4mre.php
     year={2013}
 }
 """
-import os
-import xml.etree.ElementTree as ET
-from best_download import download_file
-from lm_eval.base import MultipleChoiceTask
 
 
 class QA4MRE(MultipleChoiceTask):

lm_eval/tasks/qasper.py

@@ -9,7 +9,18 @@ text. The questions are then answered by a separate set of NLP practitioners who
 provide supporting evidence to answers.
 
 Homepage: https://allenai.org/data/qasper
+"""
+from collections import Counter
+from math import exp
+import random
+import re
+import string
+from lm_eval.base import rf
+from lm_eval.metrics import f1_score, mean
+from .common import HFTask
 
+
+_CITATION = """
 @article{DBLP:journals/corr/abs-2105-03011,
     author    = {Pradeep Dasigi and
                Kyle Lo and
@@ -30,14 +41,6 @@ Homepage: https://allenai.org/data/qasper
     bibsource = {dblp computer science bibliography, https://dblp.org}
 }
 """
-from collections import Counter
-from math import exp
-import random
-import re
-import string
-from lm_eval.base import rf
-from lm_eval.metrics import f1_score, mean
-from .common import HFTask
 
 
 def normalize_answer(s):

lm_eval/tasks/quac.py

@@ -9,7 +9,14 @@ questions to learn as much as possible about a hidden Wikipedia text, and (2)
 a teacher who answers the questions by providing short excerpts (spans) from the text.
 
 Homepage: https://quac.ai/
+"""
+import json
+import os
+from lm_eval.base import Task
+from ..utils import sh
+
 
+_CITATION = """
 @article{choi2018quac,
     title={Quac: Question answering in context},
     author={Choi, Eunsol and He, He and Iyyer, Mohit and Yatskar, Mark and Yih, Wen-tau and Choi, Yejin and Liang, Percy and Zettlemoyer, Luke},
@@ -18,11 +25,6 @@ Homepage: https://quac.ai/
 }
 """
 
-import json
-import os
-from lm_eval.base import Task
-from ..utils import sh
-
 
 class QuAC(Task):
     VERSION = 0

lm_eval/tasks/race.py

@@ -8,7 +8,16 @@ in China, which are designed for middle school and high school students. The dat
 can be served as the training and test sets for machine comprehension.
 
 Homepage: https://www.cs.cmu.edu/~glai1/data/race/
+"""
+import collections
+import datasets
+import numpy as np
+from lm_eval.base import rf
+from ..metrics import mean
+from . common import HFTask
 
+
+_CITATION = """
 @article{lai2017large,
     title={RACE: Large-scale ReAding Comprehension Dataset From Examinations},
     author={Lai, Guokun and Xie, Qizhe and Liu, Hanxiao and Yang, Yiming and Hovy, Eduard},
@@ -16,12 +25,6 @@ Homepage: https://www.cs.cmu.edu/~glai1/data/race/
     year={2017}
 }
 """
-import collections
-import datasets
-import numpy as np
-from lm_eval.base import rf
-from ..metrics import mean
-from . common import HFTask
 
 
 class each:

lm_eval/tasks/sat.py

@@ -6,7 +6,12 @@ SAT (Scholastic Aptitude Test) Analogy Questions is a dataset comprising 374
 multiple-choice analogy questions; 5 choices per question.
 
 Homepage: https://aclweb.org/aclwiki/SAT_Analogy_Questions_(State_of_the_art)
+"""
+import os
+from lm_eval.base import MultipleChoiceTask
 
+
+_CITATION = """
 @article{article,
     author = {Turney, Peter},
     year = {2006},
@@ -18,8 +23,6 @@ Homepage: https://aclweb.org/aclwiki/SAT_Analogy_Questions_(State_of_the_art)
     doi = {10.1162/coli.2006.32.3.379}
 }
 """
-import os
-from lm_eval.base import MultipleChoiceTask
 
 
 class SATAnalogies(MultipleChoiceTask):    

lm_eval/tasks/sciq.py

@@ -8,7 +8,15 @@ with 4 answer options each. For the majority of the questions, an additional par
 with supporting evidence for the correct answer is provided.
 
 Homepage: https://allenai.org/data/sciq
+"""
+import os
+import json
+import zipfile
+from lm_eval.base import MultipleChoiceTask
+from best_download import download_file
 
+
+_CITATION = """
 @inproceedings{Welbl2017CrowdsourcingMC,
     title={Crowdsourcing Multiple Choice Science Questions},
     author={Johannes Welbl and Nelson F. Liu and Matt Gardner},
@@ -16,11 +24,6 @@ Homepage: https://allenai.org/data/sciq
     year={2017}
 }
 """
-import os
-import json
-import zipfile
-from lm_eval.base import MultipleChoiceTask
-from best_download import download_file
 
 
 class SciQ(MultipleChoiceTask):

lm_eval/tasks/squad.py

@@ -12,7 +12,17 @@ To do well on SQuAD2.0, systems must not only answer questions when possible, bu
 also determine when no answer is supported by the paragraph and abstain from answering.
 
 Homepage: https://rajpurkar.github.io/SQuAD-explorer/
+"""
+import datasets
+from math import exp
+from lm_eval.base import rf
+from lm_eval.metrics import f1_score, mean
+from . common import HFTask
+from functools import partial
+from packaging import version
 
+
+_CITATION = """
 @misc{rajpurkar2018know,
     title={Know What You Don't Know: Unanswerable Questions for SQuAD}, 
     author={Pranav Rajpurkar and Robin Jia and Percy Liang},
@@ -22,13 +32,6 @@ Homepage: https://rajpurkar.github.io/SQuAD-explorer/
     primaryClass={cs.CL}
 }
 """
-import datasets
-from math import exp
-from lm_eval.base import rf
-from lm_eval.metrics import f1_score, mean
-from . common import HFTask
-from functools import partial
-from packaging import version
 
 
 def _squad_metric(predictions, references):

lm_eval/tasks/storycloze.py

@@ -7,7 +7,12 @@ understanding, story generation, and script learning. This test requires a syste
 to choose the correct ending to a four-sentence story.
 
 Homepage: https://cs.rochester.edu/nlp/rocstories/
+"""
+import csv
+from lm_eval.base import Task
 
+
+_CITATION = """
 @inproceedings{sharma-etal-2018-tackling,
     title = "Tackling the Story Ending Biases in The Story Cloze Test",
     author = "Sharma, Rishi  and
@@ -25,8 +30,6 @@ Homepage: https://cs.rochester.edu/nlp/rocstories/
     abstract = "The Story Cloze Test (SCT) is a recent framework for evaluating story comprehension and script learning. There have been a variety of models tackling the SCT so far. Although the original goal behind the SCT was to require systems to perform deep language understanding and commonsense reasoning for successful narrative understanding, some recent models could perform significantly better than the initial baselines by leveraging human-authorship biases discovered in the SCT dataset. In order to shed some light on this issue, we have performed various data analysis and analyzed a variety of top performing models presented for this task. Given the statistics we have aggregated, we have designed a new crowdsourcing scheme that creates a new SCT dataset, which overcomes some of the biases. We benchmark a few models on the new dataset and show that the top-performing model on the original SCT dataset fails to keep up its performance. Our findings further signify the importance of benchmarking NLP systems on various evolving test sets.",
 }
 """
-import csv
-from lm_eval.base import Task
 
 
 class StoryCloze(Task):

lm_eval/tasks/superglue.py

@@ -8,7 +8,17 @@ understanding tasks.
 Homepage: https://super.gluebenchmark.com/
 
 TODO: WSC requires free-form generation.
+"""
+import numpy as np
+import sklearn
+import transformers.data.metrics.squad_metrics as squad_metrics
+from . common import HFTask, yesno
+from lm_eval.base import rf
+from ..metrics import mean, acc_all, metric_max_over_ground_truths
+from ..utils import general_detokenize
 
+
+_CITATION = """
 @inproceedings{NEURIPS2019_4496bf24,
     author = {Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel},
     booktitle = {Advances in Neural Information Processing Systems},
@@ -21,13 +31,6 @@ TODO: WSC requires free-form generation.
     year = {2019}
 }
 """
-import numpy as np
-import sklearn
-import transformers.data.metrics.squad_metrics as squad_metrics
-from . common import HFTask, yesno
-from lm_eval.base import rf
-from ..metrics import mean, acc_all, metric_max_over_ground_truths
-from ..utils import general_detokenize
 
 
 class BoolQ(HFTask):

lm_eval/tasks/translation.py

@@ -8,7 +8,16 @@ here to the SacreBLEU repo from which we've obtained the datasets:
 https://github.com/mjpost/sacrebleu/blob/master/sacrebleu/dataset.py
 
 Homepage: https://github.com/mjpost/sacrebleu/blob/master/sacrebleu/dataset.py
+"""
+import pycountry
+from pprint import pprint
+from sacrebleu import sacrebleu
+from lm_eval import metrics
+from lm_eval.base import Task, rf
+from typing import List
 
+
+_CITATION = """
 @inproceedings{post-2018-call,
     title = "A Call for Clarity in Reporting {BLEU} Scores",
     author = "Post, Matt",
@@ -21,12 +30,6 @@ Homepage: https://github.com/mjpost/sacrebleu/blob/master/sacrebleu/dataset.py
     pages = "186--191",
 }
 """
-import pycountry
-from pprint import pprint
-from sacrebleu import sacrebleu
-from lm_eval import metrics
-from lm_eval.base import Task, rf
-from typing import List
 
 
 sacrebleu_datasets = sacrebleu.DATASETS

lm_eval/tasks/triviaqa.py

@@ -8,7 +8,17 @@ and independently gathered evidence documents, six per question on average, that
 high quality distant supervision for answering the questions.
 
 Homepage: https://nlp.cs.washington.edu/triviaqa/
+"""
+import os
+import json
+import jsonlines
+from lm_eval.base import Task, rf
+from ..metrics import mean
+from ..utils import sh
+from best_download import download_file
 
+
+_CITATION = """
 @InProceedings{JoshiTriviaQA2017,
     author = {Joshi, Mandar and Choi, Eunsol and Weld, Daniel S. and Zettlemoyer, Luke},
     title = {TriviaQA: A Large Scale Distantly Supervised Challenge Dataset for Reading Comprehension},
@@ -19,13 +29,6 @@ Homepage: https://nlp.cs.washington.edu/triviaqa/
     publisher = {Association for Computational Linguistics},
 }
 """
-import os
-import json
-import jsonlines
-from lm_eval.base import Task, rf
-from ..metrics import mean
-from ..utils import sh
-from best_download import download_file
 
 
 class TriviaQA(Task):

lm_eval/tasks/truthfulqa.py

@@ -18,15 +18,6 @@ https://github.com/sylinrl/TruthfulQA#Fine-tuning-GPT-3-for-evaluation. Maybe
 we could try this?
 
 Homepage: https://github.com/sylinrl/TruthfulQA
-
-@misc{lin2021truthfulqa,
-      title={TruthfulQA: Measuring How Models Mimic Human Falsehoods},
-      author={Stephanie Lin and Jacob Hilton and Owain Evans},
-      year={2021},
-      eprint={2109.07958},
-      archivePrefix={arXiv},
-      primaryClass={cs.CL}
-}
 """
 import csv
 import json
@@ -40,6 +31,18 @@ from ..metrics import mean
 from datasets import load_metric
 
 
+_CITATION = """
+@misc{lin2021truthfulqa,
+    title={TruthfulQA: Measuring How Models Mimic Human Falsehoods},
+    author={Stephanie Lin and Jacob Hilton and Owain Evans},
+    year={2021},
+    eprint={2109.07958},
+    archivePrefix={arXiv},
+    primaryClass={cs.CL}
+}
+"""
+
+
 # The default QA preset prompt for all models.
 QA_PROMPT = (
     "Q: What is human life expectancy in the United States?\n"