Skip to content

GitLab

Commit 1f8a8c1d authored by jon-tow's avatar jon-tow
Browse files

Merge branch 'master' of https://github.com/EleutherAI/lm-evaluation-harness into remove-dataset

parents b4c0275d b0acb337
Show whitespace changes
Inline Side-by-side
Showing with 207 additions and 125 deletions
lm_eval/datasets/headqa/headqa.py
View file @ 1f8a8c1d
# coding=utf-8
# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
#
# Licensed under the Apache License, Version 2.0 (the "License");
......@@ -65,8 +64,12 @@ class HeadQA(datasets.GeneratorBasedBuilder):
VERSION = datasets.Version("1.1.0")
BUILDER_CONFIGS = [
datasets.BuilderConfig(name="es", version=VERSION, description="Spanish HEAD dataset"),
datasets.BuilderConfig(name="en", version=VERSION, description="English HEAD dataset"),
datasets.BuilderConfig(
name="es", version=VERSION, description="Spanish HEAD dataset"
),
datasets.BuilderConfig(
name="en", version=VERSION, description="English HEAD dataset"
),
]
DEFAULT_CONFIG_NAME = "es"
......@@ -106,15 +109,24 @@ class HeadQA(datasets.GeneratorBasedBuilder):
return [
datasets.SplitGenerator(
name=datasets.Split.TRAIN,
gen_kwargs={"data_dir": data_dir, "filepath": os.path.join(data_lang_dir, f"train_{dir}.json")},
gen_kwargs={
"data_dir": data_dir,
"filepath": os.path.join(data_lang_dir, f"train_{dir}.json"),
},
),
datasets.SplitGenerator(
name=datasets.Split.TEST,
gen_kwargs={"data_dir": data_dir, "filepath": os.path.join(data_lang_dir, f"test_{dir}.json")},
gen_kwargs={
"data_dir": data_dir,
"filepath": os.path.join(data_lang_dir, f"test_{dir}.json"),
},
),
datasets.SplitGenerator(
name=datasets.Split.VALIDATION,
gen_kwargs={"data_dir": data_dir, "filepath": os.path.join(data_lang_dir, f"dev_{dir}.json")},
gen_kwargs={
"data_dir": data_dir,
"filepath": os.path.join(data_lang_dir, f"dev_{dir}.json"),
},
),
]
......@@ -134,7 +146,9 @@ class HeadQA(datasets.GeneratorBasedBuilder):
aids = [answer["aid"] for answer in question["answers"]]
atexts = [answer["atext"].strip() for answer in question["answers"]]
answers = [{"aid": aid, "atext": atext} for aid, atext in zip(aids, atexts)]
answers = [
{"aid": aid, "atext": atext} for aid, atext in zip(aids, atexts)
]
id_ = f"{exam_id}_{qid}"
yield id_, {
......
lm_eval/datasets/hendrycks_ethics/hendrycks_ethics.py
View file @ 1f8a8c1d
......@@ -71,54 +71,64 @@ class HendrycksEthics(datasets.GeneratorBasedBuilder):
EthicsConfig(
name="commonsense",
prefix="cm",
features=datasets.Features({
features=datasets.Features(
{
"label": datasets.Value("int32"),
"input": datasets.Value("string"),
"is_short": datasets.Value("bool"),
"edited": datasets.Value("bool"),
}),
description="The Commonsense subset contains examples focusing on moral standards and principles that most people intuitively accept."
}
),
description="The Commonsense subset contains examples focusing on moral standards and principles that most people intuitively accept.",
),
EthicsConfig(
name="deontology",
prefix="deontology",
features=datasets.Features({
features=datasets.Features(
{
"group_id": datasets.Value("int32"),
"label": datasets.Value("int32"),
"scenario": datasets.Value("string"),
"excuse": datasets.Value("string"),
}),
}
),
description="The Deontology subset contains examples focusing on whether an act is required, permitted, or forbidden according to a set of rules or constraints",
),
EthicsConfig(
name="justice",
prefix="justice",
features=datasets.Features({
features=datasets.Features(
{
"group_id": datasets.Value("int32"),
"label": datasets.Value("int32"),
"scenario": datasets.Value("string"),
}),
}
),
description="The Justice subset contains examples focusing on how a character treats another person",
),
EthicsConfig(
name="utilitarianism",
prefix="util",
features=datasets.Features({
features=datasets.Features(
{
"activity": datasets.Value("string"),
"baseline": datasets.Value("string"),
"rating": datasets.Value("string"), # Empty rating.
}),
}
),
description="The Utilitarianism subset contains scenarios that should be ranked from most pleasant to least pleasant for the person in the scenario",
),
EthicsConfig(
name="virtue",
prefix="virtue",
features=datasets.Features({
features=datasets.Features(
{
"group_id": datasets.Value("int32"),
"label": datasets.Value("int32"),
"scenario": datasets.Value("string"),
"trait": datasets.Value("string"),
}),
}
),
description="The Virtue subset contains scenarios focusing on whether virtues or vices are being exemplified",
),
]
......@@ -140,7 +150,12 @@ class HendrycksEthics(datasets.GeneratorBasedBuilder):
name=datasets.Split.TRAIN,
# These kwargs will be passed to _generate_examples
gen_kwargs={
"filepath": os.path.join(data_dir, "ethics", self.config.name, f"{self.config.prefix}_train.csv"),
"filepath": os.path.join(
data_dir,
"ethics",
self.config.name,
f"{self.config.prefix}_train.csv",
),
"split": "train",
},
),
......@@ -148,18 +163,22 @@ class HendrycksEthics(datasets.GeneratorBasedBuilder):
name=datasets.Split.TEST,
# These kwargs will be passed to _generate_examples
gen_kwargs={
"filepath": os.path.join(data_dir, "ethics", self.config.name, f"{self.config.prefix}_test.csv"),
"split": "test"
"filepath": os.path.join(
data_dir,
"ethics",
self.config.name,
f"{self.config.prefix}_test.csv",
),
"split": "test",
},
)
),
]
# method parameters are unpacked from `gen_kwargs` as given in `_split_generators`
def _generate_examples(self, filepath, split):
with open(filepath, newline='') as f:
with open(filepath, newline="") as f:
if self.config.name == "utilitarianism":
contents = csv.DictReader(
f, fieldnames=['activity', "baseline"])
contents = csv.DictReader(f, fieldnames=["activity", "baseline"])
else:
contents = csv.DictReader(f)
# For subsets with grouped scenarios, tag them with an id.
......
lm_eval/datasets/hendrycks_math/hendrycks_math.py
View file @ 1f8a8c1d
......@@ -44,13 +44,13 @@ _LICENSE = ""
_URLS = "https://people.eecs.berkeley.edu/~hendrycks/MATH.tar"
_NAMES = [
'algebra',
'counting_and_probability',
'geometry',
'intermediate_algebra',
'number_theory',
'prealgebra',
'precalculus',
"algebra",
"counting_and_probability",
"geometry",
"intermediate_algebra",
"number_theory",
"prealgebra",
"precalculus",
]
......@@ -89,7 +89,9 @@ class HendrycksMath(datasets.GeneratorBasedBuilder):
name=datasets.Split.TRAIN,
# These kwargs will be passed to _generate_examples
gen_kwargs={
"basepath": os.path.join(data_dir, "MATH", "train", self.config.name),
"basepath": os.path.join(
data_dir, "MATH", "train", self.config.name
),
"split": "train",
},
),
......@@ -97,8 +99,10 @@ class HendrycksMath(datasets.GeneratorBasedBuilder):
name=datasets.Split.TEST,
# These kwargs will be passed to _generate_examples
gen_kwargs={
"basepath": os.path.join(data_dir, "MATH", "test", self.config.name),
"split": "test"
"basepath": os.path.join(
data_dir, "MATH", "test", self.config.name
),
"split": "test",
},
),
]
......@@ -107,7 +111,7 @@ class HendrycksMath(datasets.GeneratorBasedBuilder):
def _generate_examples(self, basepath, split):
key = 0
for file in sorted(pathlib.Path(basepath).iterdir()):
with open(file, "r", encoding='utf-8') as f:
with open(file, "r", encoding="utf-8") as f:
data = json.load(f)
yield key, {
"problem": data["problem"],
......
lm_eval/datasets/lambada/lambada.py
View file @ 1f8a8c1d
......@@ -62,12 +62,34 @@ class Lambada(datasets.GeneratorBasedBuilder):
VERSION = datasets.Version("0.0.1")
BUILDER_CONFIGS = [
datasets.BuilderConfig(name="original", version=VERSION, description="The LAMBADA dataset"),
datasets.BuilderConfig(name="en", version=VERSION, description="The English translated LAMBADA dataset"),
datasets.BuilderConfig(name="fr", version=VERSION, description="The French translated LAMBADA dataset"),
datasets.BuilderConfig(name="de", version=VERSION, description="The German translated LAMBADA dataset"),
datasets.BuilderConfig(name="it", version=VERSION, description="The Italian translated LAMBADA dataset"),
datasets.BuilderConfig(name="es", version=VERSION, description="The Spanish translated LAMBADA dataset"),
datasets.BuilderConfig(
name="original", version=VERSION, description="The LAMBADA dataset"
),
datasets.BuilderConfig(
name="en",
version=VERSION,
description="The English translated LAMBADA dataset",
),
datasets.BuilderConfig(
name="fr",
version=VERSION,
description="The French translated LAMBADA dataset",
),
datasets.BuilderConfig(
name="de",
version=VERSION,
description="The German translated LAMBADA dataset",
),
datasets.BuilderConfig(
name="it",
version=VERSION,
description="The Italian translated LAMBADA dataset",
),
datasets.BuilderConfig(
name="es",
version=VERSION,
description="The Spanish translated LAMBADA dataset",
),
]
DEFAULT_CONFIG_NAME = "original"
......@@ -105,6 +127,4 @@ class Lambada(datasets.GeneratorBasedBuilder):
with open(filepath, encoding="utf-8") as f:
for key, row in enumerate(f):
data = json.loads(row)
yield key, {
"text": data["text"]
}
yield key, {"text": data["text"]}
lm_eval/datasets/logiqa/logiqa.py
View file @ 1f8a8c1d
......@@ -54,7 +54,9 @@ class Logiqa(datasets.GeneratorBasedBuilder):
VERSION = datasets.Version("0.0.1")
BUILDER_CONFIGS = [
datasets.BuilderConfig(name="logiqa", version=VERSION, description="The LogiQA dataset."),
datasets.BuilderConfig(
name="logiqa", version=VERSION, description="The LogiQA dataset."
),
]
def _info(self):
......@@ -63,9 +65,7 @@ class Logiqa(datasets.GeneratorBasedBuilder):
"label": datasets.Value("string"),
"context": datasets.Value("string"),
"question": datasets.Value("string"),
"options": datasets.features.Sequence(
datasets.Value("string")
),
"options": datasets.features.Sequence(datasets.Value("string")),
}
)
return datasets.DatasetInfo(
......@@ -77,7 +77,11 @@ class Logiqa(datasets.GeneratorBasedBuilder):
)
def _split_generators(self, dl_manager):
urls = {"train": _URLS["train"], "test": _URLS["test"], "validation": _URLS["validation"]}
urls = {
"train": _URLS["train"],
"test": _URLS["test"],
"validation": _URLS["validation"],
}
data_dir = dl_manager.download_and_extract(urls)
return [
datasets.SplitGenerator(
......@@ -91,10 +95,7 @@ class Logiqa(datasets.GeneratorBasedBuilder):
datasets.SplitGenerator(
name=datasets.Split.TEST,
# These kwargs will be passed to _generate_examples
gen_kwargs={
"filepath": data_dir["test"],
"split": "test"
},
gen_kwargs={"filepath": data_dir["test"], "split": "test"},
),
datasets.SplitGenerator(
name=datasets.Split.VALIDATION,
......@@ -110,6 +111,7 @@ class Logiqa(datasets.GeneratorBasedBuilder):
def _generate_examples(self, filepath, split):
def normalize(text):
return text.replace(".", ". ").strip()
with open(filepath, encoding="utf-8") as f:
data = f.read().strip().split("\n\n")
for key, row in enumerate(data):
......
lm_eval/datasets/mutual/mutual.py
View file @ 1f8a8c1d
......@@ -50,8 +50,14 @@ class Mutual(datasets.GeneratorBasedBuilder):
VERSION = datasets.Version("0.0.1")
BUILDER_CONFIGS = [
datasets.BuilderConfig(name="mutual", version=VERSION, description="The MuTual dataset."),
datasets.BuilderConfig(name="mutual_plus", version=VERSION, description="MuTualPlus is a more difficult MuTual that replaces positive responses with a safe responses."),
datasets.BuilderConfig(
name="mutual", version=VERSION, description="The MuTual dataset."
),
datasets.BuilderConfig(
name="mutual_plus",
version=VERSION,
description="MuTualPlus is a more difficult MuTual that replaces positive responses with a safe responses.",
),
]
def _info(self):
......@@ -79,7 +85,9 @@ class Mutual(datasets.GeneratorBasedBuilder):
name=datasets.Split.TRAIN,
# These kwargs will be passed to _generate_examples
gen_kwargs={
"basepath": os.path.join(data_dir, "MuTual-master", "data", self.config.name, "train"),
"basepath": os.path.join(
data_dir, "MuTual-master", "data", self.config.name, "train"
),
"split": "train",
},
),
......@@ -87,7 +95,9 @@ class Mutual(datasets.GeneratorBasedBuilder):
name=datasets.Split.TEST,
# These kwargs will be passed to _generate_examples
gen_kwargs={
"basepath": os.path.join(data_dir, "MuTual-master", "data", self.config.name, "test"),
"basepath": os.path.join(
data_dir, "MuTual-master", "data", self.config.name, "test"
),
"split": "test",
},
),
......@@ -95,7 +105,9 @@ class Mutual(datasets.GeneratorBasedBuilder):
name=datasets.Split.VALIDATION,
# These kwargs will be passed to _generate_examples
gen_kwargs={
"basepath": os.path.join(data_dir, "MuTual-master", "data", self.config.name, "dev"),
"basepath": os.path.join(
data_dir, "MuTual-master", "data", self.config.name, "dev"
),
"split": "dev",
},
),
......@@ -109,7 +121,7 @@ class Mutual(datasets.GeneratorBasedBuilder):
for file in sorted(Path(basepath).iterdir()):
if file.suffix != ".txt":
continue
with open(file, "r", encoding='utf-8') as f:
with open(file, "r", encoding="utf-8") as f:
data_str = f.read()
# Ignore the occasional empty file.
if not data_str:
......
lm_eval/datasets/pile/pile.py
View file @ 1f8a8c1d
......@@ -103,10 +103,7 @@ class Pile(datasets.GeneratorBasedBuilder):
datasets.SplitGenerator(
name=datasets.Split.TEST,
# These kwargs will be passed to _generate_examples
gen_kwargs={
"filepath": data_dir["test"],
"split": "test"
},
gen_kwargs={"filepath": data_dir["test"], "split": "test"},
),
datasets.SplitGenerator(
name=datasets.Split.VALIDATION,
......
lm_eval/datasets/quac/quac.py
View file @ 1f8a8c1d
......@@ -54,7 +54,9 @@ class Quac(datasets.GeneratorBasedBuilder):
VERSION = datasets.Version("1.1.0")
BUILDER_CONFIGS = [
datasets.BuilderConfig(name="quac", version=VERSION, description="The QuAC dataset"),
datasets.BuilderConfig(
name="quac", version=VERSION, description="The QuAC dataset"
),
]
def _info(self):
......@@ -90,10 +92,7 @@ class Quac(datasets.GeneratorBasedBuilder):
datasets.SplitGenerator(
name=datasets.Split.VALIDATION,
# These kwargs will be passed to _generate_examples
gen_kwargs={
"filepath": data_dir["validation"],
"split": "validation"
},
gen_kwargs={"filepath": data_dir["validation"], "split": "validation"},
),
]
......@@ -105,7 +104,7 @@ class Quac(datasets.GeneratorBasedBuilder):
for row in data:
paragraph = row["paragraphs"][0]["context"].replace("CANNOTANSWER", "")
qas = row["paragraphs"][0]["qas"]
qa_pairs = [(qa['question'], qa['answers'][0]['text']) for qa in qas]
qa_pairs = [(qa["question"], qa["answers"][0]["text"]) for qa in qas]
for (question, answer) in qa_pairs:
# Yields examples as (key, example) tuples
yield key, {
......
lm_eval/datasets/sat_analogies/sat_analogies.py
View file @ 1f8a8c1d
......@@ -44,13 +44,16 @@ _LICENSE = ""
class SatAnalogies(datasets.GeneratorBasedBuilder):
""" SAT (Scholastic Aptitude Test) Analogy Questions is a dataset comprising 374 multiple-choice analogy questions. """
"""SAT (Scholastic Aptitude Test) Analogy Questions is a dataset comprising 374 multiple-choice analogy questions."""
VERSION = datasets.Version("0.0.1")
BUILDER_CONFIGS = [
datasets.BuilderConfig(name="sat_analogies", version=VERSION,
description="The SAT Analogy Questions dataset"),
datasets.BuilderConfig(
name="sat_analogies",
version=VERSION,
description="The SAT Analogy Questions dataset",
),
]
@property
......@@ -58,7 +61,7 @@ class SatAnalogies(datasets.GeneratorBasedBuilder):
return (
"To use SAT Analogy Questions you have to download it manually. Please "
"email Peter Turney to request the data (https://www.apperceptual.com). "
"Once you recieve a download link for the dataset, supply the local path "
"Once you receive a download link for the dataset, supply the local path "
"as the `data_dir` arg: "
"`datasets.load_dataset('sat_analogies', data_dir='path/to/folder/folder_name')`"
)
......@@ -68,9 +71,7 @@ class SatAnalogies(datasets.GeneratorBasedBuilder):
{
"source": datasets.Value("string"),
"stem": datasets.Value("string"),
"choices": datasets.features.Sequence(
datasets.Value("string")
),
"choices": datasets.features.Sequence(datasets.Value("string")),
"solution": datasets.Value("string"),
}
)
......@@ -108,7 +109,7 @@ class SatAnalogies(datasets.GeneratorBasedBuilder):
if len(line) == 0 and record:
data.append(record)
record = []
elif len(line) > 0 and line[0] == '#':
elif len(line) > 0 and line[0] == "#":
# Skip comments.
continue
else:
......@@ -120,8 +121,8 @@ class SatAnalogies(datasets.GeneratorBasedBuilder):
choices = record[-6:-1]
solution = record[-1]
yield key, {
'source': source,
'stem': stem,
'choices': choices,
'solution': solution,
"source": source,
"stem": stem,
"choices": choices,
"solution": solution,
}
lm_eval/datasets/triviaqa/triviaqa.py
View file @ 1f8a8c1d
......@@ -50,13 +50,14 @@ _URLS = "http://eaidata.bmk.sh/data/triviaqa-unfiltered.tar.gz"
class Triviaqa(datasets.GeneratorBasedBuilder):
""" TriviaQA is a reading comprehension dataset containing over 650K question-answer-evidence triples """
"""TriviaQA is a reading comprehension dataset containing over 650K question-answer-evidence triples"""
VERSION = datasets.Version("0.0.1")
BUILDER_CONFIGS = [
datasets.BuilderConfig(
name="triviaqa", version=VERSION, description="The TriviaQA dataset"),
name="triviaqa", version=VERSION, description="The TriviaQA dataset"
),
]
def _info(self):
......@@ -69,7 +70,7 @@ class Triviaqa(datasets.GeneratorBasedBuilder):
"aliases": datasets.features.Sequence(
datasets.Value("string"),
),
"value": datasets.Value("string")
"value": datasets.Value("string"),
},
"search_results": datasets.features.Sequence(
{
......@@ -120,12 +121,24 @@ class Triviaqa(datasets.GeneratorBasedBuilder):
for search_result in data["SearchResults"]:
search_results.append(
{
"description": search_result["Description"] if "Description" in search_result else "",
"filename": search_result["Filename"] if "Filename" in search_result else "",
"rank": search_result["Rank"] if "Rank" in search_result else -1,
"title": search_result["Title"] if "Title" in search_result else "",
"url": search_result["Url"] if "Url" in search_result else "",
"search_context": search_result["SearchContext"] if "SearchContext" in search_result else "",
"description": search_result["Description"]
if "Description" in search_result
else "",
"filename": search_result["Filename"]
if "Filename" in search_result
else "",
"rank": search_result["Rank"]
if "Rank" in search_result
else -1,
"title": search_result["Title"]
if "Title" in search_result
else "",
"url": search_result["Url"]
if "Url" in search_result
else "",
"search_context": search_result["SearchContext"]
if "SearchContext" in search_result
else "",
}
)
yield key, {
......
lm_eval/datasets/unscramble/unscramble.py
View file @ 1f8a8c1d
......@@ -64,8 +64,9 @@ class Unscramble(datasets.GeneratorBasedBuilder):
VERSION = datasets.Version("0.0.1")
BUILDER_CONFIGS = [
datasets.BuilderConfig(name=name, version=version,
description=_DESCRIPTIONS[name])
datasets.BuilderConfig(
name=name, version=version, description=_DESCRIPTIONS[name]
)
for name, version in zip(_NAMES, [VERSION] * len(_NAMES))
]
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment