Skip to content

GitLab

Commit 63001b68 authored by bzantium's avatar bzantium
Browse files

fix conflict for upstream

parents a956bc63 84ef60ee
Hide whitespace changes
Inline Side-by-side
Showing with 1573 additions and 308 deletions
lm_eval/tasks/truthfulqa.py
View file @ 63001b68
......@@ -19,16 +19,22 @@ we could try this?
Homepage: https://github.com/sylinrl/TruthfulQA
"""
import inspect
import numpy as np
import sacrebleu
import datasets
import lm_eval.datasets.truthfulqa.truthfulqa
from rouge_score import rouge_scorer, scoring
from lm_eval.base import rf, Task
from lm_eval.metrics import mean
try:
import bleurt
HAS_BLEURT = True
except ImportError:
HAS_BLEURT = False
_CITATION = """
@misc{lin2021truthfulqa,
title={TruthfulQA: Measuring How Models Mimic Human Falsehoods},
......@@ -60,7 +66,7 @@ QA_PROMPT = (
class TruthfulQAMultipleChoice(Task):
VERSION = 1
DATASET_PATH = inspect.getfile(lm_eval.datasets.truthfulqa.truthfulqa)
DATASET_PATH = "truthful_qa"
DATASET_NAME = "multiple_choice"
def has_training_docs(self):
......@@ -82,22 +88,29 @@ class TruthfulQAMultipleChoice(Task):
raise NotImplementedError()
def doc_to_text(self, doc):
return QA_PROMPT + "\n\nQ: " + doc['question'] + "\nA:"
return QA_PROMPT + "\n\nQ: " + doc["question"] + "\nA:"
def should_decontaminate(self):
return True
def doc_to_decontamination_query(self, doc):
return doc["question"]
def doc_to_target(self, doc):
return " "
def fewshot_context(self, doc, num_fewshot, provide_description=None, rnd=None, description=None):
assert num_fewshot == 0, "TruthfulQA is intended only for the zero-shot setting."
def fewshot_context(
self, doc, num_fewshot, provide_description=None, rnd=None, description=None
):
assert (
num_fewshot == 0
), "TruthfulQA is intended only for the zero-shot setting."
return super().fewshot_context(
doc=doc,
num_fewshot=num_fewshot,
rnd=rnd,
description=description
doc=doc, num_fewshot=num_fewshot, rnd=rnd, description=description
)
def construct_requests(self, doc, ctx):
""" Uses RequestFactory to construct Requests and returns an iterable of
"""Uses RequestFactory to construct Requests and returns an iterable of
Requests which will be sent to the LM.
:param doc:
......@@ -107,11 +120,15 @@ class TruthfulQAMultipleChoice(Task):
language description, as well as the few shot examples, and the question
part of the document for `doc`.
"""
def get_lls(targets):
return [rf.loglikelihood(ctx, " " + t)[0] for t in targets]
# MC1 and MC2 targets are not always the same set of strings so we collect
# likelihoods separately for simpler processing.
return get_lls(doc['mc1_targets']["choices"]) + get_lls(doc['mc2_targets']["choices"])
return get_lls(doc["mc1_targets"]["choices"]) + get_lls(
doc["mc2_targets"]["choices"]
)
def process_results(self, doc, results):
"""Take a single document and the LM results and evaluates, returning a
......@@ -123,46 +140,44 @@ class TruthfulQAMultipleChoice(Task):
:param results:
The results of the requests created in construct_requests.
"""
def mc1(lls):
# The gold answers in `mc1_targets` are always first (index = `0`).
return np.argmax(lls) == 0
def mc2(lls):
# Split on the first `0` as everything before it is true (`1`).
split_idx = list(doc['mc2_targets']["labels"]).index(0)
split_idx = list(doc["mc2_targets"]["labels"]).index(0)
# Compute the normalized probability mass for the correct answer.
ll_true, ll_false = lls[:split_idx], lls[split_idx:]
p_true, p_false = np.exp(np.array(ll_true)), np.exp(np.array(ll_false))
p_true = p_true / (sum(p_true) + sum(p_false))
return sum(p_true)
split_idx = len(doc['mc1_targets']["choices"])
split_idx = len(doc["mc1_targets"]["choices"])
mc1_lls, mc2_lls = results[:split_idx], results[split_idx:]
return {
"mc1": mc1(mc1_lls),
"mc2": mc2(mc2_lls)
}
return {"mc1": mc1(mc1_lls), "mc2": mc2(mc2_lls)}
def aggregation(self):
return {
"mc1": mean,
"mc2": mean
}
return {"mc1": mean, "mc2": mean}
def higher_is_better(self):
return {
"mc1": True,
"mc2": True
}
return {"mc1": True, "mc2": True}
class TruthfulQAGeneration(Task):
VERSION = 1
DATASET_PATH = inspect.getfile(lm_eval.datasets.truthfulqa.truthfulqa)
DATASET_PATH = "truthful_qa"
DATASET_NAME = "generation"
def __init__(self):
super().__init__()
if not HAS_BLEURT:
raise ImportError(
"`TruthfulQAGeneration` requires the `bleurt` package. Please install it with:\n"
"pip install bleurt@https://github.com/google-research/bleurt/archive/b610120347ef22b494b6d69b4316e303f5932516.zip#egg=bleurt"
"\nWARNING: Installing any other version of bleurt may result in different results."
)
self.bleurt = datasets.load_metric("bleurt")
def has_training_docs(self):
......@@ -183,44 +198,45 @@ class TruthfulQAGeneration(Task):
answer = answer.strip()
if len(answer):
# Add a period after all answers.
if answer[-1] != '.':
formatted_answers.append(answer + '.')
if answer[-1] != ".":
formatted_answers.append(answer + ".")
else:
formatted_answers.append(answer)
return formatted_answers
def validation_docs(self):
for doc in self.dataset["validation"]:
incorrect_answers = self._format_answers(doc['incorrect_answers'])
correct_answers = self._format_answers(doc['correct_answers'])
incorrect_answers = self._format_answers(doc["incorrect_answers"])
correct_answers = self._format_answers(doc["correct_answers"])
if "I have no comment." not in correct_answers:
correct_answers.append("I have no comment.")
yield {
'question': doc['question'].strip(),
'correct_answers': correct_answers,
'incorrect_answers': incorrect_answers
"question": doc["question"].strip(),
"correct_answers": correct_answers,
"incorrect_answers": incorrect_answers,
}
def test_docs(self):
raise NotImplementedError()
def doc_to_text(self, doc):
return QA_PROMPT + "\n\nQ: " + doc['question']
return QA_PROMPT + "\n\nQ: " + doc["question"]
def doc_to_target(self, doc):
return " "
def fewshot_context(self, doc, num_fewshot, provide_description=None, rnd=None, description=None):
assert num_fewshot == 0, "TruthfulQA is intended only for the zero-shot setting."
def fewshot_context(
self, doc, num_fewshot, provide_description=None, rnd=None, description=None
):
assert (
num_fewshot == 0
), "TruthfulQA is intended only for the zero-shot setting."
return super().fewshot_context(
doc=doc,
num_fewshot=num_fewshot,
rnd=rnd,
description=description
doc=doc, num_fewshot=num_fewshot, rnd=rnd, description=description
)
def construct_requests(self, doc, ctx):
""" Uses RequestFactory to construct Requests and returns an iterable of
"""Uses RequestFactory to construct Requests and returns an iterable of
Requests which will be sent to the LM.
:param doc:
......@@ -231,7 +247,7 @@ class TruthfulQAGeneration(Task):
part of the document for `doc`.
"""
# TODO: Find a way to cap the number of generated tokens to `50` as in the official implementation.
completion = rf.greedy_until(ctx, ['.'])
completion = rf.greedy_until(ctx, {"until": ["."]})
return completion
def process_results(self, doc, results):
......@@ -245,18 +261,18 @@ class TruthfulQAGeneration(Task):
The results of the requests created in construct_requests.
"""
completion = results[0].strip()
true_refs, false_refs = doc['correct_answers'], doc['incorrect_answers']
true_refs, false_refs = doc["correct_answers"], doc["incorrect_answers"]
all_refs = true_refs + false_refs
# Process the sentence-level BLEURT, BLEU, and ROUGE for similarity measures.
# BLEURT
bleurt_scores_true = self.bleurt.compute(
predictions=[completion] * len(true_refs),
references=true_refs)['scores']
predictions=[completion] * len(true_refs), references=true_refs
)["scores"]
bleurt_scores_false = self.bleurt.compute(
predictions=[completion] * len(false_refs),
references=false_refs)['scores']
predictions=[completion] * len(false_refs), references=false_refs
)["scores"]
bleurt_correct = max(bleurt_scores_true)
bleurt_incorrect = max(bleurt_scores_false)
bleurt_max = bleurt_correct
......@@ -265,8 +281,8 @@ class TruthfulQAGeneration(Task):
# BLEU
bleu_scores = [self.bleu([[ref]], [completion]) for ref in all_refs]
bleu_correct = np.nanmax(bleu_scores[:len(true_refs)])
bleu_incorrect = np.nanmax(bleu_scores[len(true_refs):])
bleu_correct = np.nanmax(bleu_scores[: len(true_refs)])
bleu_incorrect = np.nanmax(bleu_scores[len(true_refs) :])
bleu_max = bleu_correct
bleu_diff = bleu_correct - bleu_incorrect
bleu_acc = int(bleu_correct > bleu_incorrect)
......@@ -274,23 +290,23 @@ class TruthfulQAGeneration(Task):
# ROUGE-N
rouge_scores = [self.rouge([ref], [completion]) for ref in all_refs]
# ROUGE-1
rouge1_scores = [score['rouge1'] for score in rouge_scores]
rouge1_correct = np.nanmax(rouge1_scores[:len(true_refs)])
rouge1_incorrect = np.nanmax(rouge1_scores[len(true_refs):])
rouge1_scores = [score["rouge1"] for score in rouge_scores]
rouge1_correct = np.nanmax(rouge1_scores[: len(true_refs)])
rouge1_incorrect = np.nanmax(rouge1_scores[len(true_refs) :])
rouge1_max = rouge1_correct
rouge1_diff = rouge1_correct - rouge1_incorrect
rouge1_acc = int(rouge1_correct > rouge1_incorrect)
# ROUGE-2
rouge2_scores = [score['rouge2'] for score in rouge_scores]
rouge2_correct = np.nanmax(rouge2_scores[:len(true_refs)])
rouge2_incorrect = np.nanmax(rouge2_scores[len(true_refs):])
rouge2_scores = [score["rouge2"] for score in rouge_scores]
rouge2_correct = np.nanmax(rouge2_scores[: len(true_refs)])
rouge2_incorrect = np.nanmax(rouge2_scores[len(true_refs) :])
rouge2_max = rouge2_correct
rouge2_diff = rouge2_correct - rouge2_incorrect
rouge2_acc = int(rouge2_correct > rouge2_incorrect)
# ROUGE-L
rougeL_scores = [score['rougeLsum'] for score in rouge_scores]
rougeL_correct = np.nanmax(rougeL_scores[:len(true_refs)])
rougeL_incorrect = np.nanmax(rougeL_scores[len(true_refs):])
rougeL_scores = [score["rougeLsum"] for score in rouge_scores]
rougeL_correct = np.nanmax(rougeL_scores[: len(true_refs)])
rougeL_incorrect = np.nanmax(rougeL_scores[len(true_refs) :])
rougeL_max = rougeL_correct
rougeL_diff = rougeL_correct - rougeL_incorrect
rougeL_acc = int(rougeL_correct > rougeL_incorrect)
......@@ -299,19 +315,15 @@ class TruthfulQAGeneration(Task):
"bleurt_max": bleurt_max,
"bleurt_acc": bleurt_acc,
"bleurt_diff": bleurt_diff,
"bleu_max": bleu_max,
"bleu_acc": bleu_acc,
"bleu_diff": bleu_diff,
"rouge1_max": rouge1_max,
"rouge1_acc": rouge1_acc,
"rouge1_diff": rouge1_diff,
"rouge2_max": rouge2_max,
"rouge2_acc": rouge2_acc,
"rouge2_diff": rouge2_diff,
"rougeL_max": rougeL_max,
"rougeL_acc": rougeL_acc,
"rougeL_diff": rougeL_diff,
......@@ -322,19 +334,15 @@ class TruthfulQAGeneration(Task):
"bleurt_max": mean,
"bleurt_acc": mean,
"bleurt_diff": mean,
"bleu_max": mean,
"bleu_acc": mean,
"bleu_diff": mean,
"rouge1_max": mean,
"rouge1_acc": mean,
"rouge1_diff": mean,
"rouge2_max": mean,
"rouge2_acc": mean,
"rouge2_diff": mean,
"rougeL_max": mean,
"rougeL_acc": mean,
"rougeL_diff": mean,
......@@ -345,19 +353,15 @@ class TruthfulQAGeneration(Task):
"bleurt_max": True,
"bleurt_acc": True,
"bleurt_diff": True,
"bleu_max": True,
"bleu_acc": True,
"bleu_diff": True,
"rouge1_max": True,
"rouge1_acc": True,
"rouge1_diff": True,
"rouge2_max": True,
"rouge2_acc": True,
"rouge2_diff": True,
"rougeL_max": True,
"rougeL_acc": True,
"rougeL_diff": True,
......@@ -381,7 +385,7 @@ class TruthfulQAGeneration(Task):
force=False,
lowercase=False,
tokenize="intl",
use_effective_order=False
use_effective_order=False,
).score
return score
......@@ -398,9 +402,11 @@ class TruthfulQAGeneration(Task):
rouge_types = ["rouge1", "rouge2", "rougeLsum"]
scorer = rouge_scorer.RougeScorer(rouge_types)
# Add newlines between sentences to correctly compute `rougeLsum`.
def _prepare_summary(summary):
summary = summary.replace(" . ", ".\n")
return summary
# Accumulate confidence intervals.
aggregator = scoring.BootstrapAggregator()
for ref, pred in zip(refs, preds):
......@@ -408,4 +414,4 @@ class TruthfulQAGeneration(Task):
pred = _prepare_summary(pred)
aggregator.add_scores(scorer.score(ref, pred))
result = aggregator.aggregate()
return {type: result[type].mid.fmeasure*100 for type in rouge_types}
return {type: result[type].mid.fmeasure * 100 for type in rouge_types}
lm_eval/tasks/unscramble.py
View file @ 63001b68
......@@ -49,29 +49,29 @@ class WordUnscrambleTask(Task):
def doc_to_text(self, doc):
return doc["context"]
def should_decontaminate(self):
return True
def doc_to_decontamination_query(self, doc):
return doc["context"]
def doc_to_target(self, doc):
return doc["completion"]
def construct_requests(self, doc, ctx):
completion = rf.greedy_until(ctx, ["\n"])
completion = rf.greedy_until(ctx, {"until": ["\n"]})
return completion
def process_results(self, doc, results):
pred = results[0]
gold = doc["completion"]
return {
"acc": int(pred == gold)
}
return {"acc": int(pred == gold)}
def aggregation(self):
return {
"acc": mean
}
return {"acc": mean}
def higher_is_better(self):
return {
"acc": True
}
return {"acc": True}
class Anagrams1(WordUnscrambleTask):
......
lm_eval/tasks/webqs.py
View file @ 63001b68
......@@ -54,14 +54,20 @@ class WebQs(Task):
return self.dataset["test"]
def doc_to_text(self, doc):
return "Question: " + doc['question'] + '\nAnswer:'
return "Question: " + doc["question"] + "\nAnswer:"
def should_decontaminate(self):
return True
def doc_to_decontamination_query(self, doc):
return doc["question"]
def doc_to_target(self, doc):
# this picks one answer to be the "correct" one, despite sometimes
# this picks one answer to be the "correct" one, despite sometimes
# multiple correct answers being possible.
# TODO: make sure we're actually handling multi-answer correctly
return " " + doc['answers'][0]
return " " + doc["answers"][0]
def _remove_prefixes(self, aliases):
# Optimization: Remove any alias that has a strict prefix elsewhere in the list
# we can do this because if the prefix is acceptable by isgreedy, we can stop looking
......@@ -75,15 +81,13 @@ class WebQs(Task):
def construct_requests(self, doc, ctx):
ret = []
for alias in self._remove_prefixes(doc['answers']):
for alias in self._remove_prefixes(doc["answers"]):
_, is_prediction = rf.loglikelihood(ctx, " " + alias)
ret.append(is_prediction)
return ret
def process_results(self, doc, results):
return {
"acc": float(any(results))
}
return {"acc": float(any(results))}
def aggregation(self):
return {
......@@ -91,6 +95,4 @@ class WebQs(Task):
}
def higher_is_better(self):
return {
"acc": True
}
return {"acc": True}
lm_eval/tasks/wikitext.py
View file @ 63001b68
......@@ -2,7 +2,7 @@
Pointer Sentinel Mixture Models
https://arxiv.org/pdf/1609.07843.pdf
The WikiText language modeling dataset is a collection of over 100 million tokens
The WikiText language modeling dataset is a collection of over 100 million tokens
extracted from the set of verified Good and Featured articles on Wikipedia.
NOTE: This `Task` is based on WikiText-2.
......@@ -10,14 +10,12 @@ NOTE: This `Task` is based on WikiText-2.
Homepage: https://www.salesforce.com/products/einstein/ai-research/the-wikitext-dependency-language-modeling-dataset/
"""
import re
import inspect
import lm_eval.datasets.wikitext.wikitext
from lm_eval.base import PerplexityTask
_CITATION = """
@misc{merity2016pointer,
title={Pointer Sentinel Mixture Models},
title={Pointer Sentinel Mixture Models},
author={Stephen Merity and Caiming Xiong and James Bradbury and Richard Socher},
year={2016},
eprint={1609.07843},
......@@ -63,7 +61,7 @@ def wikitext_detokenizer(string):
class WikiText(PerplexityTask):
VERSION = 1
DATASET_PATH = inspect.getfile(lm_eval.datasets.wikitext.wikitext)
DATASET_PATH = "EleutherAI/wikitext_document_level"
DATASET_NAME = "wikitext-2-raw-v1"
def has_training_docs(self):
......@@ -76,20 +74,23 @@ class WikiText(PerplexityTask):
return True
def training_docs(self):
return map(self._load_doc, self.dataset["train"])
return map(self._process_doc, self.dataset["train"])
def validation_docs(self):
return map(self._load_doc, self.dataset["validation"])
return map(self._process_doc, self.dataset["validation"])
def test_docs(self):
return map(self._load_doc, self.dataset["test"])
return map(self._process_doc, self.dataset["test"])
def _load_doc(self, doc):
def _process_doc(self, doc):
return doc["page"]
def doc_to_target(self, doc):
return wikitext_detokenizer(doc)
def should_decontaminate(self):
return True
def count_words(self, doc):
# count number of words in *original doc before detokenization*
return len(re.split(r"\s+", doc))
lm_eval/tasks/winogrande.py
View file @ 63001b68
"""
WinoGrande: An Adversarial Winograd Schema Challenge at Scale
https://arxiv.org/pdf/1907.10641.pdf
WinoGrande is a collection of 44k problems, inspired by Winograd Schema Challenge
(Levesque, Davis, and Morgenstern 2011), but adjusted to improve the scale and
robustness against the dataset-specific bias. Formulated as a fill-in-a-blank
task with binary options, the goal is to choose the right option for a given
sentence which requires commonsense reasoning.
NOTE: This evaluation of Winogrande uses partial evaluation as described by
Trinh & Le in Simple Method for Commonsense Reasoning (2018).
See: https://arxiv.org/abs/1806.02847
Homepage: https://leaderboard.allenai.org/winogrande/submissions/public
"""
import numpy as np
from lm_eval.base import rf, Task
from lm_eval.metrics import mean
_CITATION = """
@article{sakaguchi2019winogrande,
title={WinoGrande: An Adversarial Winograd Schema Challenge at Scale},
author={Sakaguchi, Keisuke and Bras, Ronan Le and Bhagavatula, Chandra and Choi, Yejin},
journal={arXiv preprint arXiv:1907.10641},
year={2019}
}
"""
class Winogrande(Task):
VERSION = 0
DATASET_PATH = "winogrande"
DATASET_NAME = "winogrande_xl"
answer_to_num = {'1': 0, '2': 1}
def has_training_docs(self):
return True
def has_validation_docs(self):
return True
def has_test_docs(self):
return False
def training_docs(self):
if self._training_docs is None:
self._training_docs = list(self.dataset["train"])
return self._training_docs
def validation_docs(self):
return self.dataset["validation"]
def doc_to_text(self, doc):
return self.partial_context(doc, doc["option" + doc["answer"]])
@classmethod
def partial_context(cls, doc, option):
# Substitute the pronoun in the sentence with the specified option
# and ignore everything after.
pronoun_loc = doc["sentence"].index("_")
return doc["sentence"][:pronoun_loc] + option
def doc_to_target(self, doc):
return self.partial_target(doc)
@classmethod
def partial_target(cls, doc):
# The target is everything after the document specified pronoun.
pronoun_loc = doc["sentence"].index("_") + 1
return " " + doc["sentence"][pronoun_loc:].strip()
def construct_requests(self, doc, ctx):
"""Uses RequestFactory to construct Requests and returns an iterable of
Requests which will be sent to the LM.
:param doc:
The document as returned from training_docs, validation_docs, or test_docs.
:param ctx: str
The context string, generated by fewshot_context. This includes the natural
language description, as well as the few shot examples, and the question
part of the document for `doc`.
"""
target = self.partial_target(doc)
lls = []
for option in [doc["option1"], doc["option2"]]:
partial_ctx = self.partial_context(doc, option)
full_ctx = self.append_context(ctx, partial_ctx)
lls.append(rf.loglikelihood(full_ctx, target)[0])
return lls
@classmethod
def append_context(cls, ctx, partial_ctx):
ctx = ctx.split("\n\n") # Each fewshot context is on its own new line.
ctx.pop() # Remove the correct context put in by `doc_to_text`.
return "\n\n".join([*ctx, partial_ctx]) if ctx else partial_ctx
def process_results(self, doc, results):
"""Take a single document and the LM results and evaluates, returning a
dict where keys are the names of submetrics and values are the values of
the metric for that one document
:param doc:
The document as returned from training_docs, validation_docs, or test_docs.
:param results:
The results of the requests created in construct_requests.
"""
return {
"acc": np.argmax(results) == self.answer_to_num[doc["answer"]]
}
def aggregation(self):
"""
:returns: {str: [float] -> float}
A dictionary where keys are the names of submetrics and values are
functions that aggregate a list of metrics
"""
return {
"acc": mean
}
def higher_is_better(self):
"""
:returns: {str: bool}
A dictionary where keys are the names of submetrics and values are
whether a higher value of the submetric is better
"""
return {
"acc": True
}
"""
WinoGrande: An Adversarial Winograd Schema Challenge at Scale
https://arxiv.org/pdf/1907.10641.pdf
WinoGrande is a collection of 44k problems, inspired by Winograd Schema Challenge
(Levesque, Davis, and Morgenstern 2011), but adjusted to improve the scale and
robustness against the dataset-specific bias. Formulated as a fill-in-a-blank
task with binary options, the goal is to choose the right option for a given
sentence which requires commonsense reasoning.
NOTE: This evaluation of Winogrande uses partial evaluation as described by
Trinh & Le in Simple Method for Commonsense Reasoning (2018).
See: https://arxiv.org/abs/1806.02847
Homepage: https://leaderboard.allenai.org/winogrande/submissions/public
"""
import numpy as np
from lm_eval.base import rf, Task
from lm_eval.metrics import mean
_CITATION = """
@article{sakaguchi2019winogrande,
title={WinoGrande: An Adversarial Winograd Schema Challenge at Scale},
author={Sakaguchi, Keisuke and Bras, Ronan Le and Bhagavatula, Chandra and Choi, Yejin},
journal={arXiv preprint arXiv:1907.10641},
year={2019}
}
"""
class Winogrande(Task):
VERSION = 0
DATASET_PATH = "winogrande"
DATASET_NAME = "winogrande_xl"
answer_to_num = {"1": 0, "2": 1}
def has_training_docs(self):
return True
def has_validation_docs(self):
return True
def has_test_docs(self):
return False
def training_docs(self):
if self._training_docs is None:
self._training_docs = list(self.dataset["train"])
return self._training_docs
def validation_docs(self):
return self.dataset["validation"]
def doc_to_text(self, doc):
return self.partial_context(doc, doc["option" + doc["answer"]])
def should_decontaminate(self):
return True
def doc_to_decontamination_query(self, doc):
return doc["sentence"]
@classmethod
def partial_context(cls, doc, option):
# Substitute the pronoun in the sentence with the specified option
# and ignore everything after.
pronoun_loc = doc["sentence"].index("_")
return doc["sentence"][:pronoun_loc] + option
def doc_to_target(self, doc):
return self.partial_target(doc)
@classmethod
def partial_target(cls, doc):
# The target is everything after the document specified pronoun.
pronoun_loc = doc["sentence"].index("_") + 1
return " " + doc["sentence"][pronoun_loc:].strip()
def construct_requests(self, doc, ctx):
"""Uses RequestFactory to construct Requests and returns an iterable of
Requests which will be sent to the LM.
:param doc:
The document as returned from training_docs, validation_docs, or test_docs.
:param ctx: str
The context string, generated by fewshot_context. This includes the natural
language description, as well as the few shot examples, and the question
part of the document for `doc`.
"""
target = self.partial_target(doc)
lls = []
for option in [doc["option1"], doc["option2"]]:
partial_ctx = self.partial_context(doc, option)
full_ctx = self.append_context(ctx, partial_ctx)
lls.append(rf.loglikelihood(full_ctx, target)[0])
return lls
@classmethod
def append_context(cls, ctx, partial_ctx):
ctx = ctx.split("\n\n") # Each fewshot context is on its own new line.
ctx.pop() # Remove the correct context put in by `doc_to_text`.
return "\n\n".join([*ctx, partial_ctx]) if ctx else partial_ctx
def process_results(self, doc, results):
"""Take a single document and the LM results and evaluates, returning a
dict where keys are the names of submetrics and values are the values of
the metric for that one document
:param doc:
The document as returned from training_docs, validation_docs, or test_docs.
:param results:
The results of the requests created in construct_requests.
"""
return {"acc": np.argmax(results) == self.answer_to_num[doc["answer"]]}
def aggregation(self):
"""
:returns: {str: [float] -> float}
A dictionary where keys are the names of submetrics and values are
functions that aggregate a list of metrics
"""
return {"acc": mean}
def higher_is_better(self):
"""
:returns: {str: bool}
A dictionary where keys are the names of submetrics and values are
whether a higher value of the submetric is better
"""
return {"acc": True}
lm_eval/tasks/wsc273.py
View file @ 63001b68
......@@ -40,8 +40,19 @@ class WinogradSchemaChallenge273(Task):
DATASET_PATH = "winograd_wsc"
DATASET_NAME = "wsc273"
upper_pronouns = ["A", "An", "The", "She", "He",
"It", "They", "My", "His", "Her", "Their"]
upper_pronouns = [
"A",
"An",
"The",
"She",
"He",
"It",
"They",
"My",
"His",
"Her",
"Their",
]
def has_training_docs(self):
return False
......@@ -53,9 +64,9 @@ class WinogradSchemaChallenge273(Task):
return True
def test_docs(self):
return map(self._load_doc, self.dataset["test"])
return map(self._process_doc, self.dataset["test"])
def _load_doc(self, doc):
def _process_doc(self, doc):
# The HF implementation of `wsc273` is not `partial evaluation` friendly.
doc["text"] = doc["text"].replace(" ", " ")
doc["options"][0] = self.__normalize_option(doc, doc["options"][0])
......@@ -68,7 +79,7 @@ class WinogradSchemaChallenge273(Task):
option += "'s"
# Appropriately lowercase the pronoun in the option.
pronoun = option.split()[0]
start_of_sentence = doc["text"][doc['pronoun_loc'] - 2] == '.'
start_of_sentence = doc["text"][doc["pronoun_loc"] - 2] == "."
if not start_of_sentence and pronoun in self.upper_pronouns:
return option.replace(pronoun, pronoun.lower())
return option
......@@ -85,11 +96,17 @@ class WinogradSchemaChallenge273(Task):
def doc_to_text(self, doc):
return self.partial_context(doc, doc["options"][doc["label"]])
def should_decontaminate(self):
return True
def doc_to_decontamination_query(self, doc):
return doc["text"]
@classmethod
def partial_context(cls, doc, option):
# Substitute the pronoun in the original text with the specified
# option and ignore everything after.
return doc["text"][:doc["pronoun_loc"]] + option
return doc["text"][: doc["pronoun_loc"]] + option
def doc_to_target(self, doc):
return self.partial_target(doc)
......@@ -135,9 +152,7 @@ class WinogradSchemaChallenge273(Task):
:param results:
The results of the requests created in construct_requests.
"""
return {
"acc": np.argmax(results) == doc["label"]
}
return {"acc": np.argmax(results) == doc["label"]}
def aggregation(self):
"""
......@@ -145,9 +160,7 @@ class WinogradSchemaChallenge273(Task):
A dictionary where keys are the names of submetrics and values are
functions that aggregate a list of metrics
"""
return {
"acc": mean
}
return {"acc": mean}
def higher_is_better(self):
"""
......@@ -155,6 +168,4 @@ class WinogradSchemaChallenge273(Task):
A dictionary where keys are the names of submetrics and values are
whether a higher value of the submetric is better
"""
return {
"acc": True
}
return {"acc": True}
lm_eval/tasks/xcopa.py 0 → 100644
View file @ 63001b68
"""
XCOPA: A Multilingual Dataset for Causal Commonsense Reasoning
https://ducdauge.github.io/files/xcopa.pdf
The Cross-lingual Choice of Plausible Alternatives dataset is a benchmark to evaluate the ability of machine learning models to transfer commonsense reasoning across languages.
The dataset is the translation and reannotation of the English COPA (Roemmele et al. 2011) and covers 11 languages from 11 families and several areas around the globe.
The dataset is challenging as it requires both the command of world knowledge and the ability to generalise to new languages.
All the details about the creation of XCOPA and the implementation of the baselines are available in the paper.
Homepage: https://github.com/cambridgeltl/xcopa
"""
from .superglue import Copa
_CITATION = """
@inproceedings{ponti2020xcopa,
title={{XCOPA: A} Multilingual Dataset for Causal Commonsense Reasoning},
author={Edoardo M. Ponti, Goran Glava\v{s}, Olga Majewska, Qianchu Liu, Ivan Vuli\'{c} and Anna Korhonen},
booktitle={Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
year={2020},
url={https://ducdauge.github.io/files/xcopa.pdf}
}
"""
class XCopa(Copa):
VERSION = 0
DATASET_PATH = "xcopa"
DATASET_NAME = None
CAUSE = "because"
EFFECT = "therefore"
def has_training_docs(self):
return False
def has_validation_docs(self):
return True
def has_test_docs(self):
return True
def validation_docs(self):
return self.dataset["validation"]
def test_docs(self):
return self.dataset["test"]
def doc_to_text(self, doc):
# Drop the period
connector = {
"cause": self.CAUSE,
"effect": self.EFFECT,
}[doc["question"]]
return doc["premise"].strip()[:-1] + f" {connector}"
class XCopaEt(XCopa):
DATASET_NAME = "et"
CAUSE = "sest"
EFFECT = "seetõttu"
class XCopaHt(XCopa):
DATASET_NAME = "ht"
CAUSE = "poukisa"
EFFECT = "donk sa"
class XCopaIt(XCopa):
DATASET_NAME = "it"
CAUSE = "perché"
EFFECT = "quindi"
class XCopaId(XCopa):
DATASET_NAME = "id"
CAUSE = "karena"
EFFECT = "maka"
class XCopaQu(XCopa):
DATASET_NAME = "qu"
CAUSE = "imataq"
EFFECT = "chaymi"
class XCopaSw(XCopa):
DATASET_NAME = "sw"
CAUSE = "kwa sababu"
EFFECT = "kwa hiyo"
class XCopaZh(XCopa):
DATASET_NAME = "zh"
CAUSE = "因为"
EFFECT = "所以"
class XCopaTa(XCopa):
DATASET_NAME = "ta"
CAUSE = "காரணமாக"
EFFECT = "எனவே"
class XCopaTh(XCopa):
DATASET_NAME = "th"
CAUSE = "เพราะ"
EFFECT = "ดังนั้น"
class XCopaTr(XCopa):
DATASET_NAME = "tr"
CAUSE = "çünkü"
EFFECT = "bu yüzden"
class XCopaVi(XCopa):
DATASET_NAME = "vi"
CAUSE = "bởi vì"
EFFECT = "vì vậy"
LANGS = ["et", "ht", "it", "id", "qu", "sw", "zh", "ta", "th", "tr", "vi"]
LANG_CLASSES = [
XCopaEt,
XCopaHt,
XCopaIt,
XCopaId,
XCopaQu,
XCopaSw,
XCopaZh,
XCopaTa,
XCopaTh,
XCopaTr,
XCopaVi,
]
def construct_tasks():
tasks = {}
for lang, lang_class in zip(LANGS, LANG_CLASSES):
tasks[f"xcopa_{lang}"] = lang_class
return tasks
lm_eval/tasks/xnli.py 0 → 100644
View file @ 63001b68
"""
XNLI: Evaluating Cross-lingual Sentence Representations
https://arxiv.org/abs/1809.05053
Based on the implementation of @yongzx (see https://github.com/EleutherAI/lm-evaluation-harness/pull/258)
Prompt format (same as XGLM and mGPT):
sentence1 + ", right? " + mask = (Yes|Also|No) + ", " + sentence2
Predicition is the full sequence with the highest likelihood.
Language specific prompts are translated word-by-word with Google Translate
and may differ from the ones used by mGPT and XGLM (they do not provide their prompts).
Homepage: https://github.com/facebookresearch/XNLI
"""
import numpy as np
from lm_eval.base import rf, Task
from lm_eval.metrics import mean
_CITATIONS = """
@InProceedings{conneau2018xnli,
author = "Conneau, Alexis
and Rinott, Ruty
and Lample, Guillaume
and Williams, Adina
and Bowman, Samuel R.
and Schwenk, Holger
and Stoyanov, Veselin",
title = "XNLI: Evaluating Cross-lingual Sentence Representations",
booktitle = "Proceedings of the 2018 Conference on Empirical Methods
in Natural Language Processing",
year = "2018",
publisher = "Association for Computational Linguistics",
location = "Brussels, Belgium",
}
"""
class XNLIBase(Task):
VERSION = 0
DATASET_PATH = "xnli"
DATASET_NAME = None
QUESTION_WORD = None # 'right'
ENTAILMENT_LABEL = None # 'Yes'
NEUTRAL_LABEL = None # 'Also'
CONTRADICTION_LABEL = None # 'No'
def has_training_docs(self):
return True
def has_validation_docs(self):
return True
def has_test_docs(self):
return True
def training_docs(self):
return self.dataset["train"]
def validation_docs(self):
return self.dataset["validation"]
def test_docs(self):
return self.dataset["test"]
def doc_to_text(self, doc):
# Example:
# The girl that can help me is all the way across town, right? Yes, The girl I need help from lives a ways away.
# [MASK] is replaced with ENTAILMENT_LABEL, NEUTRAL_LABEL, or CONTRADICTION_LABEL
return (
doc["premise"]
+ ", "
+ self.QUESTION_WORD
+ "? [MASK], "
+ doc["hypothesis"]
)
def doc_to_target(self, doc):
# True = entailment
# False = contradiction
# Neither = neutral
return (
" "
+ [self.ENTAILMENT_LABEL, self.NEUTRAL_LABEL, self.CONTRADICTION_LABEL][
doc["label"]
]
)
def construct_requests(self, doc, ctx):
"""Uses RequestFactory to construct Requests and returns an iterable of
Requests which will be sent to the LM.
:param doc:
The document as returned from training_docs, validation_docs, or test_docs.
:param ctx: str
The context string, generated by fewshot_context. This includes the natural
language description, as well as the few shot examples, and the question
part of the document for `doc`.
"""
ll_true = rf.loglikelihood_rolling(ctx.replace("[MASK]", self.ENTAILMENT_LABEL))
ll_neither = rf.loglikelihood_rolling(ctx.replace("[MASK]", self.NEUTRAL_LABEL))
ll_false = rf.loglikelihood_rolling(
ctx.replace("[MASK]", self.CONTRADICTION_LABEL)
)
return ll_true, ll_neither, ll_false
def process_results(self, doc, results):
"""Take a single document and the LM results and evaluates, returning a
dict where keys are the names of submetrics and values are the values of
the metric for that one document
:param doc:
The document as returned from training_docs, validation_docs, or test_docs.
:param results:
The results of the requests created in construct_requests.
"""
gold = doc["label"]
pred = np.argmax(results)
return {"acc": pred == gold}
def aggregation(self):
"""
:returns: {str: [float] -> float}
A dictionary where keys are the names of submetrics and values are
functions that aggregate a list of metrics
"""
return {"acc": mean}
def higher_is_better(self):
"""
:returns: {str: bool}
A dictionary where keys are the names of submetrics and values are
whether a higher value of the submetric is better
"""
return {"acc": True}
class XNLI_en(XNLIBase): # English
DATASET_NAME = "en"
QUESTION_WORD = "right"
ENTAILMENT_LABEL = "Yes"
NEUTRAL_LABEL = "Also"
CONTRADICTION_LABEL = "No"
class XNLI_de(XNLIBase): # German
DATASET_NAME = "de"
QUESTION_WORD = "richtig"
ENTAILMENT_LABEL = "Ja"
NEUTRAL_LABEL = "Auch"
CONTRADICTION_LABEL = "Nein"
class XNLI_ar(XNLIBase): # Arabic
DATASET_NAME = "ar"
QUESTION_WORD = "صحيح"
ENTAILMENT_LABEL = "نعم"
NEUTRAL_LABEL = "لذا"
CONTRADICTION_LABEL = "رقم"
class XNLI_bg(XNLIBase): # Bulgarian
DATASET_NAME = "bg"
QUESTION_WORD = "правилно"
ENTAILMENT_LABEL = "да"
NEUTRAL_LABEL = "така"
CONTRADICTION_LABEL = "не"
class XNLI_el(XNLIBase): # Greek
DATASET_NAME = "el"
QUESTION_WORD = "σωστός"
ENTAILMENT_LABEL = "Ναί"
NEUTRAL_LABEL = "Έτσι"
CONTRADICTION_LABEL = "όχι"
class XNLI_es(XNLIBase): # Spanish
DATASET_NAME = "es"
QUESTION_WORD = "correcto"
ENTAILMENT_LABEL = "Sí"
NEUTRAL_LABEL = "Asi que"
CONTRADICTION_LABEL = "No"
class XNLI_fr(XNLIBase): # French
DATASET_NAME = "fr"
QUESTION_WORD = "correct"
ENTAILMENT_LABEL = "Oui"
NEUTRAL_LABEL = "Aussi"
CONTRADICTION_LABEL = "Non"
class XNLI_hi(XNLIBase): # Hindi
DATASET_NAME = "hi"
QUESTION_WORD = "सही"
ENTAILMENT_LABEL = "हाँ"
NEUTRAL_LABEL = "इसलिए"
CONTRADICTION_LABEL = "नहीं"
class XNLI_ru(XNLIBase): # Russian
DATASET_NAME = "ru"
QUESTION_WORD = "правильно"
ENTAILMENT_LABEL = "Да"
NEUTRAL_LABEL = "Так"
CONTRADICTION_LABEL = "Нет"
class XNLI_sw(XNLIBase): # Swahili
DATASET_NAME = "sw"
QUESTION_WORD = "sahihi"
ENTAILMENT_LABEL = "Ndiyo"
NEUTRAL_LABEL = "Hivyo"
CONTRADICTION_LABEL = "Hapana"
class XNLI_th(XNLIBase): # Thai
DATASET_NAME = "th"
QUESTION_WORD = "ถูกต้อง"
ENTAILMENT_LABEL = "ใช่"
NEUTRAL_LABEL = "ดังนั้น"
CONTRADICTION_LABEL = "ไม่"
class XNLI_tr(XNLIBase): # Turkish
DATASET_NAME = "tr"
QUESTION_WORD = "doğru"
ENTAILMENT_LABEL = "Evet"
NEUTRAL_LABEL = "Böylece"
CONTRADICTION_LABEL = "Hayır"
class XNLI_ur(XNLIBase): # Urdu
DATASET_NAME = "ur"
QUESTION_WORD = "صحیح"
ENTAILMENT_LABEL = "جی ہاں"
NEUTRAL_LABEL = "اس لئے"
CONTRADICTION_LABEL = "نہیں"
class XNLI_vi(XNLIBase): # Vietnamese
DATASET_NAME = "vi"
QUESTION_WORD = "đúng"
ENTAILMENT_LABEL = "Vâng"
NEUTRAL_LABEL = "Vì vậy"
CONTRADICTION_LABEL = "Không"
class XNLI_zh(XNLIBase): # Chinese
DATASET_NAME = "zh"
QUESTION_WORD = "正确"
ENTAILMENT_LABEL = "是的"
NEUTRAL_LABEL = "所以"
CONTRADICTION_LABEL = "不是的"
LANGS = [
"ar",
"bg",
"de",
"el",
"en",
"es",
"fr",
"hi",
"ru",
"sw",
"th",
"tr",
"ur",
"vi",
"zh",
]
LANG_CLASSES = [
XNLI_ar,
XNLI_bg,
XNLI_de,
XNLI_el,
XNLI_en,
XNLI_es,
XNLI_fr,
XNLI_hi,
XNLI_ru,
XNLI_sw,
XNLI_th,
XNLI_tr,
XNLI_ur,
XNLI_vi,
XNLI_zh,
]
def construct_tasks():
tasks = {}
for lang, lang_class in zip(LANGS, LANG_CLASSES):
tasks[f"xnli_{lang}"] = lang_class
return tasks
lm_eval/tasks/xstorycloze.py 0 → 100644
View file @ 63001b68
"""
Few-shot Learning with Multilingual Language Models
https://arxiv.org/abs/2112.10668
XStoryCloze consists of the professionally translated version of the [English StoryCloze dataset](https://cs.rochester.edu/nlp/rocstories/) (Spring 2016 version) to 10 non-English languages. This dataset is released by Meta AI.
Homepage: https://github.com/facebookresearch/fairseq/pull/4820
"""
from .storycloze import StoryCloze
_CITATION = """
@article{DBLP:journals/corr/abs-2112-10668,
author = {Xi Victoria Lin and
Todor Mihaylov and
Mikel Artetxe and
Tianlu Wang and
Shuohui Chen and
Daniel Simig and
Myle Ott and
Naman Goyal and
Shruti Bhosale and
Jingfei Du and
Ramakanth Pasunuru and
Sam Shleifer and
Punit Singh Koura and
Vishrav Chaudhary and
Brian O'Horo and
Jeff Wang and
Luke Zettlemoyer and
Zornitsa Kozareva and
Mona T. Diab and
Veselin Stoyanov and
Xian Li},
title = {Few-shot Learning with Multilingual Language Models},
journal = {CoRR},
volume = {abs/2112.10668},
year = {2021},
url = {https://arxiv.org/abs/2112.10668},
eprinttype = {arXiv},
eprint = {2112.10668},
timestamp = {Tue, 04 Jan 2022 15:59:27 +0100},
biburl = {https://dblp.org/rec/journals/corr/abs-2112-10668.bib},
bibsource = {dblp computer science bibliography, https://dblp.org}
}
"""
_LANG = ["en", "ru", "zh", "es", "ar", "hi", "id", "te", "sw", "eu", "my"]
def create_all_tasks():
"""Creates a dictionary of tasks from a list of subjects
:return: {task_name: task}
"""
return {f"xstory_cloze_{lang}": create_task(lang) for lang in _LANG}
def create_task(lang):
class XStoryCloze(StoryCloze):
DATASET_PATH = "juletxara/xstory_cloze"
DATASET_NAME = lang
def __init__(self):
super().__init__(data_dir="")
def has_training_docs(self):
return True
def has_validation_docs(self):
return True
def has_test_docs(self):
return False
def training_docs(self):
return self.dataset["train"]
def validation_docs(self):
return self.dataset["eval"]
def test_docs(self):
pass
return XStoryCloze
lm_eval/tasks/xwinograd.py 0 → 100644
View file @ 63001b68
"""
It's All in the Heads: Using Attention Heads as a Baseline for Cross-Lingual Transfer in Commonsense Reasoning
https://arxiv.org/abs/2106.12066
Multilingual winograd schema challenge that includes English, French, Japanese, Portuguese, Russian and Chinese. Winograd schema challenges come from the XWinograd dataset introduced in Tikhonov et al. As it only contains 16 Chinese schemas, we add 488 Chinese schemas from clue/cluewsc2020.
Homepage: https://huggingface.co/datasets/Muennighoff/xwinograd
"""
from .winogrande import Winogrande
_CITATION = """
@misc{muennighoff2022crosslingual,
title={Crosslingual Generalization through Multitask Finetuning},
author={Niklas Muennighoff and Thomas Wang and Lintang Sutawika and Adam Roberts and Stella Biderman and Teven Le Scao and M Saiful Bari and Sheng Shen and Zheng-Xin Yong and Hailey Schoelkopf and Xiangru Tang and Dragomir Radev and Alham Fikri Aji and Khalid Almubarak and Samuel Albanie and Zaid Alyafeai and Albert Webson and Edward Raff and Colin Raffel},
year={2022},
eprint={2211.01786},
archivePrefix={arXiv},
primaryClass={cs.CL}
}
@misc{tikhonov2021heads,
title={It's All in the Heads: Using Attention Heads as a Baseline for Cross-Lingual Transfer in Commonsense Reasoning},
author={Alexey Tikhonov and Max Ryabinin},
year={2021},
eprint={2106.12066},
archivePrefix={arXiv},
primaryClass={cs.CL}
}
"""
_LANG = ["en", "fr", "jp", "pt", "ru", "zh"]
def create_all_tasks():
"""Creates a dictionary of tasks from a list of subjects
:return: {task_name: task}
"""
return {f"xwinograd_{lang}": create_task(lang) for lang in _LANG}
def create_task(lang):
class XWinograd(Winogrande):
DATASET_PATH = "Muennighoff/xwinograd"
DATASET_NAME = lang
def __init__(self):
super().__init__()
def has_training_docs(self):
return False
def has_validation_docs(self):
return False
def has_test_docs(self):
return True
def training_docs(self):
pass
def validation_docs(self):
pass
def test_docs(self):
return self.dataset["test"]
return XWinograd
lm_eval/utils.py
View file @ 63001b68
......@@ -5,8 +5,11 @@ import collections
import functools
import inspect
import sys
import pytest
from typing import List
from typing import List, Union
import torch
from omegaconf import OmegaConf
class ExitCodeError(Exception):
......@@ -18,6 +21,29 @@ def sh(x):
raise ExitCodeError()
def escaped_split(text, sep_char, maxsplit=-1):
"""Split text into a list on occurrences of the given separation
character `sep_char`. The separation character may be escaped by a
backslash to avoid splitting at that location.
The separation character must be a string of size 1.
If `maxsplit` is given, at most `maxsplit` splits are done (thus,
the list will have at most `maxsplit + 1` elements). If `maxsplit`
is not specified or less than 0, then there is no limit on the
number of splits (all possible splits are made).
"""
assert (
len(sep_char) == 1
), "separation string must be a single character for escaped splitting"
if maxsplit == 0:
return text
maxsplit = max(0, maxsplit)
return re.split(r"(?<!\\)" + sep_char, text, maxsplit)
def simple_parse_args_string(args_string):
"""
Parses something like
......@@ -28,12 +54,10 @@ def simple_parse_args_string(args_string):
if not args_string:
return {}
arg_list = args_string.split(",")
args_dict = {}
for arg in arg_list:
k, v = arg.split("=")
args_dict[k] = v
args_dict = OmegaConf.to_object(OmegaConf.from_dotlist(arg_list))
return args_dict
def join_iters(iters):
for iter in iters:
yield from iter
......@@ -46,23 +70,26 @@ def chunks(iter, n):
if len(arr) == n:
yield arr
arr = []
if arr: yield arr
if arr:
yield arr
def group(arr, fn):
res = collections.defaultdict(list)
for ob in arr:
res[fn(ob)].append(ob)
return list(res.values())
def general_detokenize(string):
string = string.replace(" n't", "n't")
string = string.replace(" )", ")")
string = string.replace("( ", "(")
string = string.replace("\" ", "\"")
string = string.replace(" \"", "\"")
string = string.replace('" ', '"')
string = string.replace(' "', '"')
string = re.sub(r" (['.,])", r"\1", string)
return string
......@@ -94,10 +121,7 @@ def get_rolling_token_windows(token_list, prefix_token, max_seq_len, context_len
# Special handling for first window: predict all tokens
first_seq_len = min(max_seq_len, len(token_list))
yield (
[prefix_token] + token_list[:first_seq_len - 1],
token_list[:first_seq_len]
)
yield ([prefix_token] + token_list[: first_seq_len - 1], token_list[:first_seq_len])
predicted += first_seq_len
while predicted < len(token_list):
......@@ -105,61 +129,84 @@ def get_rolling_token_windows(token_list, prefix_token, max_seq_len, context_len
window_end = predicted + window_pred_len
yield (
token_list[window_end - max_seq_len - 1:window_end - 1],
token_list[window_end - window_pred_len:window_end],
token_list[window_end - max_seq_len - 1 : window_end - 1],
token_list[window_end - window_pred_len : window_end],
)
predicted += window_pred_len
def make_disjoint_window(pair):
""" Takes output from get_rolling_token_windows and makes the context not overlap with the continuation """
def make_disjoint_window(pair):
"""Takes output from get_rolling_token_windows and makes the context not overlap with the continuation"""
a, b = pair
return a[: len(a) - (len(b) - 1)], b
def select_continuation_from_batch_left_padding(
generations: Union[List[List[int]], torch.Tensor], max_context_size: int
):
"""Select the continuation from the batch, removing prompts of different lengths.
Args:
generations (Union[List[List[int]], torch.Tensor]):
A tensor or list-of-lists of shape [batch_size, sequence length].
max_context_size (int):
The size of the biggest context; generations will proceed from that
index.
Example:
PAD PAD Continue : The dog chased the cat [every day of the week]
Riddle me this : The dog chased the cat [yesterday] PAD PAD PAD PAD
Output:
[every day of the week]
[yesterday] PAD PAD PAD PAD
"""
return generations[:, max_context_size:]
return a[:-(len(b) - 1)], b
class Reorderer:
def __init__(self, arr, fn):
self.size = len(arr)
arr = list(enumerate(arr))
arr = group(arr, lambda x: fn(x[1]))
arr = [
([y[0] for y in x], x[0][1]) for x in arr
]
arr = [([y[0] for y in x], x[0][1]) for x in arr]
arr.sort(key=lambda x: fn(x[1]))
self.arr = arr
def get_reordered(self):
return [x[1] for x in self.arr]
def get_original(self, newarr):
res = [None] * self.size
cov = [False] * self.size
for (inds, _), v in zip(self.arr, newarr):
for ind in inds:
for ind in inds:
res[ind] = v
cov[ind] = True
assert all(cov)
return res
def positional_deprecated(fn):
"""
A decorator to nudge users into passing only keyword args (`kwargs`) to the
A decorator to nudge users into passing only keyword args (`kwargs`) to the
wrapped function, `fn`.
"""
@functools.wraps(fn)
def _wrapper(*args, **kwargs):
if len(args) != 1 if inspect.ismethod(fn) else 0:
print(f"WARNING: using {fn.__name__} with positional arguments is "
if len(args) != 1 if inspect.ismethod(fn) else 0:
print(
f"WARNING: using {fn.__name__} with positional arguments is "
"deprecated and will be disallowed in a future version of "
"lm-evaluation-harness!")
"lm-evaluation-harness!"
)
return fn(*args, **kwargs)
return _wrapper
@positional_deprecated
def find_test_root(start_path: pathlib.Path) -> pathlib.Path:
"""
......@@ -169,22 +216,33 @@ def find_test_root(start_path: pathlib.Path) -> pathlib.Path:
cur_path = start_path.resolve()
max_layers = 3
for _ in range(max_layers):
if (cur_path / 'tests' / 'test_version_stable.py').exists():
if (cur_path / "tests" / "test_version_stable.py").exists():
return cur_path
else:
cur_path = cur_path.parent.resolve()
raise FileNotFoundError(f"Unable to find package root within {max_layers} upwards" +\
f"of {start_path}")
raise FileNotFoundError(
f"Unable to find package root within {max_layers} upwards" + f"of {start_path}"
)
@positional_deprecated
def run_task_tests(task_list: List[str]):
"""
Find the package root and run the tests for the given tasks
"""
import pytest
package_root = find_test_root(start_path=pathlib.Path(__file__))
task_string = ' or '.join(task_list)
args = [f'{package_root}/tests/test_version_stable.py', f'--rootdir={package_root}', '-k', f'{task_string}']
task_string = " or ".join(task_list)
args = [
f"{package_root}/tests/test_version_stable.py",
f"--rootdir={package_root}",
"-k",
f"{task_string}",
]
sys.path.append(str(package_root))
pytest_return_val = pytest.main(args)
if pytest_return_val:
raise ValueError(f"Not all tests for the specified tasks ({task_list}) ran successfully! Error code: {pytest_return_val}")
\ No newline at end of file
raise ValueError(
f"Not all tests for the specified tasks ({task_list}) ran successfully! Error code: {pytest_return_val}"
)
main.py
View file @ 63001b68
import argparse
import json
import logging
import fnmatch
import os
from lm_eval import tasks, evaluator
logging.getLogger("openai").setLevel(logging.WARNING)
def _is_json_task(task_name):
return task_name == "json" or task_name.startswith("json=")
class MultiChoice:
def __init__(self, choices):
self.choices = choices
# Simple wildcard support (linux filename patterns)
def __contains__(self, values):
for value in values.split(","):
if len(fnmatch.filter(self.choices, value)) == 0 and not _is_json_task(
value
):
return False
return True
def __iter__(self):
for choice in self.choices:
yield choice
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('--model', required=True)
parser.add_argument('--model_args', default="")
parser.add_argument('--tasks', default="all_tasks")
parser.add_argument('--provide_description', action="store_true")
parser.add_argument('--num_fewshot', type=int, default=0)
parser.add_argument('--batch_size', type=int, default=None)
parser.add_argument('--device', type=str, default=None)
parser.add_argument('--output_path', default=None)
parser.add_argument('--limit', type=int, default=None)
parser.add_argument('--no_cache', action="store_true")
parser.add_argument('--description_dict_path', default=None)
parser.add_argument('--check_integrity', action="store_true")
parser.add_argument("--model", required=True)
parser.add_argument("--model_args", default="")
parser.add_argument("--tasks", default=None, choices=MultiChoice(tasks.ALL_TASKS))
parser.add_argument("--provide_description", action="store_true")
parser.add_argument("--num_fewshot", type=int, default=0)
parser.add_argument("--batch_size", type=str, default=None)
parser.add_argument("--device", type=str, default=None)
parser.add_argument("--output_path", default=None)
parser.add_argument("--limit", type=float, default=None,
help="Limit the number of examples per task. "
"If <1, limit is a percentage of the total number of examples.")
parser.add_argument("--data_sampling", type=float, default=None)
parser.add_argument("--no_cache", action="store_true")
parser.add_argument("--decontamination_ngrams_path", default=None)
parser.add_argument("--description_dict_path", default=None)
parser.add_argument("--check_integrity", action="store_true")
parser.add_argument("--write_out", action="store_true", default=False)
parser.add_argument("--output_base_path", type=str, default=None)
return parser.parse_args()
# Returns a list containing all values of the source_list that
# match at least one of the patterns
def pattern_match(patterns, source_list):
task_names = set()
for pattern in patterns:
if _is_json_task(pattern):
task_names.add(pattern)
for matching in fnmatch.filter(source_list, pattern):
task_names.add(matching)
return sorted(list(task_names))
def main():
args = parse_args()
assert not args.provide_description # not implemented
if args.limit:
print("WARNING: --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.")
print(
"WARNING: --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT."
)
if args.tasks == "all_tasks":
if args.tasks is None:
task_names = tasks.ALL_TASKS
else:
task_names = args.tasks.split(",")
task_names = pattern_match(args.tasks.split(","), tasks.ALL_TASKS)
print(f"Selected Tasks: {task_names}")
description_dict = {}
if args.description_dict_path:
with open(args.description_dict_path, 'r') as f:
with open(args.description_dict_path, "r") as f:
description_dict = json.load(f)
results = evaluator.simple_evaluate(
......@@ -51,14 +101,17 @@ def main():
no_cache=args.no_cache,
limit=args.limit,
description_dict=description_dict,
check_integrity=args.check_integrity
decontamination_ngrams_path=args.decontamination_ngrams_path,
check_integrity=args.check_integrity,
write_out=args.write_out,
output_base_path=args.output_base_path,
)
dumped = json.dumps(results, indent=2)
print(dumped)
if args.output_path:
os.makedirs(os.path.dirname(args.output_path), exist_ok=True)
with open(args.output_path, "w") as f:
f.write(dumped)
......
pile_statistics.json 0 → 100644
View file @ 63001b68
{
"Data": "Pile statistics",
"Document Count": 210607728,
"Total Pile Characters": 421215456,
"File Start Offsets": [
0,
7021438,
14042822,
21066113,
28086515,
35106072,
42123306,
49145091,
56165817,
63185587,
70211208,
77234322,
84249267,
91267634,
98285983,
105305110,
112322489,
119342491,
126367373,
133389153,
140412039,
147432373,
154452516,
161470190,
168492733,
175512521,
182526939,
189547478,
196565318,
203583306
]
}
results/bloom/bloom-1b1/README.md 0 → 100644
View file @ 63001b68
# bloom-1b1
## bloom-1b1_common_sense_reasoning_0-shot.json
| Task |Version| Metric |Value| |Stderr|
|-------------|------:|--------|----:|---|-----:|
|arc_challenge| 0|acc |23.63|± | 1.24|
| | |acc_norm|25.68|± | 1.28|
|arc_easy | 0|acc |51.47|± | 1.03|
| | |acc_norm|45.45|± | 1.02|
|boolq | 1|acc |59.08|± | 0.86|
|copa | 0|acc |68.00|± | 4.69|
|hellaswag | 0|acc |34.63|± | 0.47|
| | |acc_norm|41.77|± | 0.49|
|mc_taco | 0|em |14.49| | |
| | |f1 |32.43| | |
|openbookqa | 0|acc |19.60|± | 1.78|
| | |acc_norm|29.40|± | 2.04|
|piqa | 0|acc |67.14|± | 1.10|
| | |acc_norm|67.14|± | 1.10|
|prost | 0|acc |23.41|± | 0.31|
| | |acc_norm|30.50|± | 0.34|
|swag | 0|acc |43.43|± | 0.35|
| | |acc_norm|58.28|± | 0.35|
|winogrande | 0|acc |54.93|± | 1.40|
|wsc273 | 0|acc |68.50|± | 2.82|
## bloom-1b1_gsm8k_8-shot.json
|Task |Version|Metric|Value| |Stderr|
|-----|------:|------|----:|---|-----:|
|gsm8k| 0|acc | 0.83|± | 0.25|
## bloom-1b1_mathematical_reasoning_few_shot_5-shot.json
| Task |Version| Metric |Value| |Stderr|
|-------------------------|------:|--------|----:|---|-----:|
|drop | 1|em | 1.38|± | 0.12|
| | |f1 | 4.01|± | 0.15|
|gsm8k | 0|acc | 0.00|± | 0.00|
|math_algebra | 1|acc | 0.00|± | 0.00|
|math_counting_and_prob | 1|acc | 0.21|± | 0.21|
|math_geometry | 1|acc | 0.21|± | 0.21|
|math_intermediate_algebra| 1|acc | 0.00|± | 0.00|
|math_num_theory | 1|acc | 0.19|± | 0.19|
|math_prealgebra | 1|acc | 0.11|± | 0.11|
|math_precalc | 1|acc | 0.00|± | 0.00|
|mathqa | 0|acc |23.55|± | 0.78|
| | |acc_norm|23.62|± | 0.78|
## bloom-1b1_pawsx_0-shot.json
| Task |Version|Metric|Value| |Stderr|
|--------|------:|------|----:|---|-----:|
|pawsx_de| 0|acc |46.95|± | 1.12|
|pawsx_en| 0|acc |52.45|± | 1.12|
|pawsx_es| 0|acc |51.50|± | 1.12|
|pawsx_fr| 0|acc |46.15|± | 1.11|
|pawsx_ja| 0|acc |48.40|± | 1.12|
|pawsx_ko| 0|acc |49.90|± | 1.12|
|pawsx_zh| 0|acc |48.95|± | 1.12|
## bloom-1b1_question_answering_0-shot.json
| Task |Version| Metric |Value| |Stderr|
|-------------|------:|------------|----:|---|-----:|
|headqa_en | 0|acc |26.44|± | 0.84|
| | |acc_norm |30.49|± | 0.88|
|headqa_es | 0|acc |24.43|± | 0.82|
| | |acc_norm |28.30|± | 0.86|
|logiqa | 0|acc |18.89|± | 1.54|
| | |acc_norm |25.65|± | 1.71|
|squad2 | 1|exact | 4.17| | |
| | |f1 | 6.60| | |
| | |HasAns_exact| 2.19| | |
| | |HasAns_f1 | 7.05| | |
| | |NoAns_exact | 6.14| | |
| | |NoAns_f1 | 6.14| | |
| | |best_exact |50.07| | |
| | |best_f1 |50.07| | |
|triviaqa | 1|acc | 2.68|± | 0.15|
|truthfulqa_mc| 1|mc1 |25.34|± | 1.52|
| | |mc2 |41.80|± | 1.46|
|webqs | 0|acc | 1.38|± | 0.26|
## bloom-1b1_reading_comprehension_0-shot.json
|Task|Version|Metric|Value| |Stderr|
|----|------:|------|----:|---|-----:|
|coqa| 1|f1 |45.57|± | 1.88|
| | |em |32.98|± | 1.95|
|drop| 1|em | 3.31|± | 0.18|
| | |f1 | 8.63|± | 0.22|
|race| 1|acc |32.63|± | 1.45|
## bloom-1b1_xcopa_0-shot.json
| Task |Version|Metric|Value| |Stderr|
|--------|------:|------|----:|---|-----:|
|xcopa_et| 0|acc | 50.6|± | 2.24|
|xcopa_ht| 0|acc | 53.0|± | 2.23|
|xcopa_id| 0|acc | 64.8|± | 2.14|
|xcopa_it| 0|acc | 50.8|± | 2.24|
|xcopa_qu| 0|acc | 51.2|± | 2.24|
|xcopa_sw| 0|acc | 54.4|± | 2.23|
|xcopa_ta| 0|acc | 57.0|± | 2.22|
|xcopa_th| 0|acc | 53.2|± | 2.23|
|xcopa_tr| 0|acc | 53.0|± | 2.23|
|xcopa_vi| 0|acc | 62.4|± | 2.17|
|xcopa_zh| 0|acc | 59.4|± | 2.20|
## bloom-1b1_xnli_0-shot.json
| Task |Version|Metric|Value| |Stderr|
|-------|------:|------|----:|---|-----:|
|xnli_ar| 0|acc |33.93|± | 0.67|
|xnli_bg| 0|acc |34.13|± | 0.67|
|xnli_de| 0|acc |39.64|± | 0.69|
|xnli_el| 0|acc |34.03|± | 0.67|
|xnli_en| 0|acc |51.48|± | 0.71|
|xnli_es| 0|acc |47.98|± | 0.71|
|xnli_fr| 0|acc |47.15|± | 0.71|
|xnli_hi| 0|acc |42.32|± | 0.70|
|xnli_ru| 0|acc |40.46|± | 0.69|
|xnli_sw| 0|acc |35.29|± | 0.68|
|xnli_th| 0|acc |33.75|± | 0.67|
|xnli_tr| 0|acc |34.79|± | 0.67|
|xnli_ur| 0|acc |37.33|± | 0.68|
|xnli_vi| 0|acc |44.45|± | 0.70|
|xnli_zh| 0|acc |36.23|± | 0.68|
## bloom-1b1_xstory_cloze_0-shot.json
| Task |Version|Metric|Value| |Stderr|
|---------------|------:|------|----:|---|-----:|
|xstory_cloze_ar| 0|acc |52.88|± | 1.28|
|xstory_cloze_en| 0|acc |62.54|± | 1.25|
|xstory_cloze_es| 0|acc |58.31|± | 1.27|
|xstory_cloze_eu| 0|acc |54.33|± | 1.28|
|xstory_cloze_hi| 0|acc |55.53|± | 1.28|
|xstory_cloze_id| 0|acc |57.91|± | 1.27|
|xstory_cloze_my| 0|acc |46.19|± | 1.28|
|xstory_cloze_ru| 0|acc |48.25|± | 1.29|
|xstory_cloze_sw| 0|acc |50.56|± | 1.29|
|xstory_cloze_te| 0|acc |56.39|± | 1.28|
|xstory_cloze_zh| 0|acc |58.04|± | 1.27|
## bloom-1b1_xwinograd_0-shot.json
| Task |Version|Metric|Value| |Stderr|
|------------|------:|------|----:|---|-----:|
|xwinograd_en| 0|acc |69.98|± | 0.95|
|xwinograd_fr| 0|acc |66.27|± | 5.22|
|xwinograd_jp| 0|acc |52.87|± | 1.61|
|xwinograd_pt| 0|acc |63.12|± | 2.98|
|xwinograd_ru| 0|acc |54.29|± | 2.81|
|xwinograd_zh| 0|acc |69.25|± | 2.06|
results/bloom/bloom-1b1/bloom-1b1_common_sense_reasoning_0-shot.json 0 → 100644
View file @ 63001b68
{
"results": {
"boolq": {
"acc": 0.5908256880733945,
"acc_stderr": 0.008599563442397352
},
"arc_easy": {
"acc": 0.5147306397306397,
"acc_stderr": 0.010255329977562096,
"acc_norm": 0.45454545454545453,
"acc_norm_stderr": 0.010217299762709435
},
"openbookqa": {
"acc": 0.196,
"acc_stderr": 0.017770751227744862,
"acc_norm": 0.294,
"acc_norm_stderr": 0.020395095484936614
},
"hellaswag": {
"acc": 0.3463453495319657,
"acc_stderr": 0.004748324319714264,
"acc_norm": 0.4177454690300737,
"acc_norm_stderr": 0.004921798492608764
},
"swag": {
"acc": 0.43431970408877335,
"acc_stderr": 0.0035044592489844794,
"acc_norm": 0.5828251524542637,
"acc_norm_stderr": 0.0034862531772295617
},
"arc_challenge": {
"acc": 0.2363481228668942,
"acc_stderr": 0.012414960524301834,
"acc_norm": 0.2568259385665529,
"acc_norm_stderr": 0.0127669237941168
},
"mc_taco": {
"em": 0.1448948948948949,
"f1": 0.32425976796237205
},
"wsc273": {
"acc": 0.684981684981685,
"acc_stderr": 0.028165854394193602
},
"winogrande": {
"acc": 0.5493291239147593,
"acc_stderr": 0.013983928869040239
},
"prost": {
"acc": 0.23409479077711356,
"acc_stderr": 0.003093545711826552,
"acc_norm": 0.3049743808710504,
"acc_norm_stderr": 0.003363606918420179
},
"copa": {
"acc": 0.68,
"acc_stderr": 0.04688261722621504
},
"piqa": {
"acc": 0.6713819368879217,
"acc_stderr": 0.010959127105167048,
"acc_norm": 0.6713819368879217,
"acc_norm_stderr": 0.010959127105167044
}
},
"versions": {
"boolq": 1,
"arc_easy": 0,
"openbookqa": 0,
"hellaswag": 0,
"swag": 0,
"arc_challenge": 0,
"mc_taco": 0,
"wsc273": 0,
"winogrande": 0,
"prost": 0,
"copa": 0,
"piqa": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=bigscience/bloom-1b1,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda:0",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/bloom/bloom-1b1/bloom-1b1_gsm8k_8-shot.json 0 → 100644
View file @ 63001b68
{
"results": {
"gsm8k": {
"acc": 0.008339651250947688,
"acc_stderr": 0.002504942226860508
}
},
"versions": {
"gsm8k": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=bigscience/bloom-1b1,use_accelerate=True",
"num_fewshot": 8,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/bloom/bloom-1b1/bloom-1b1_mathematical_reasoning_few_shot_5-shot.json 0 → 100644
View file @ 63001b68
{
"results": {
"mathqa": {
"acc": 0.2355108877721943,
"acc_stderr": 0.007767687364650971,
"acc_norm": 0.23618090452261306,
"acc_norm_stderr": 0.0077753193787470495
},
"gsm8k": {
"acc": 0.0,
"acc_stderr": 0.0
},
"drop": {
"em": 0.013842281879194632,
"em_stderr": 0.001196510970060749,
"f1": 0.040085989932885986,
"f1_stderr": 0.0014841664758736023
},
"math_geometry": {
"acc": 0.0020876826722338203,
"acc_stderr": 0.0020876826722338315
},
"math_counting_and_prob": {
"acc": 0.002109704641350211,
"acc_stderr": 0.002109704641350211
},
"math_prealgebra": {
"acc": 0.001148105625717566,
"acc_stderr": 0.0011481056257175708
},
"math_num_theory": {
"acc": 0.001851851851851852,
"acc_stderr": 0.0018518518518518448
},
"math_precalc": {
"acc": 0.0,
"acc_stderr": 0.0
},
"math_algebra": {
"acc": 0.0,
"acc_stderr": 0.0
},
"math_intermediate_algebra": {
"acc": 0.0,
"acc_stderr": 0.0
}
},
"versions": {
"mathqa": 0,
"gsm8k": 0,
"drop": 1,
"math_geometry": 1,
"math_counting_and_prob": 1,
"math_prealgebra": 1,
"math_num_theory": 1,
"math_precalc": 1,
"math_algebra": 1,
"math_intermediate_algebra": 1
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=bigscience/bloom-1b1,use_accelerate=True",
"num_fewshot": 5,
"batch_size": "auto",
"device": "cuda:0",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/bloom/bloom-1b1/bloom-1b1_pawsx_0-shot.json 0 → 100644
View file @ 63001b68
{
"results": {
"pawsx_es": {
"acc": 0.515,
"acc_stderr": 0.011178102477052804
},
"pawsx_zh": {
"acc": 0.4895,
"acc_stderr": 0.011180669867648657
},
"pawsx_fr": {
"acc": 0.4615,
"acc_stderr": 0.011149934327957058
},
"pawsx_ko": {
"acc": 0.499,
"acc_stderr": 0.01118311365477017
},
"pawsx_de": {
"acc": 0.4695,
"acc_stderr": 0.011162310405413175
},
"pawsx_ja": {
"acc": 0.484,
"acc_stderr": 0.011177408788874897
},
"pawsx_en": {
"acc": 0.5245,
"acc_stderr": 0.011169702598013186
}
},
"versions": {
"pawsx_es": 0,
"pawsx_zh": 0,
"pawsx_fr": 0,
"pawsx_ko": 0,
"pawsx_de": 0,
"pawsx_ja": 0,
"pawsx_en": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=bigscience/bloom-1b1",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/bloom/bloom-1b1/bloom-1b1_question_answering_0-shot.json 0 → 100644
View file @ 63001b68
{
"results": {
"truthfulqa_mc": {
"mc1": 0.2533659730722154,
"mc1_stderr": 0.01522589934082683,
"mc2": 0.4179977378869182,
"mc2_stderr": 0.014601549068840484
},
"webqs": {
"acc": 0.013779527559055118,
"acc_stderr": 0.002586718737195641
},
"logiqa": {
"acc": 0.1889400921658986,
"acc_stderr": 0.01535436463822078,
"acc_norm": 0.2565284178187404,
"acc_norm_stderr": 0.017129443327887562
},
"squad2": {
"exact": 4.169123220752969,
"f1": 6.5956997780058355,
"HasAns_exact": 2.192982456140351,
"HasAns_f1": 7.05309437656277,
"NoAns_exact": 6.139613120269134,
"NoAns_f1": 6.139613120269134,
"best_exact": 50.07159100480081,
"best_f1": 50.07159100480081
},
"headqa_es": {
"acc": 0.24434719183078046,
"acc_stderr": 0.008207488987159709,
"acc_norm": 0.2830051057622174,
"acc_norm_stderr": 0.008604004902114394
},
"headqa_en": {
"acc": 0.26440554339897887,
"acc_stderr": 0.008423643607316284,
"acc_norm": 0.30488694383661563,
"acc_norm_stderr": 0.008793112278191295
},
"triviaqa": {
"acc": 0.026783346592415803,
"acc_stderr": 0.001517985028991893
}
},
"versions": {
"truthfulqa_mc": 1,
"webqs": 0,
"logiqa": 0,
"squad2": 1,
"headqa_es": 0,
"headqa_en": 0,
"triviaqa": 1
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=bigscience/bloom-1b1,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda:0",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/bloom/bloom-1b1/bloom-1b1_reading_comprehension_0-shot.json 0 → 100644
View file @ 63001b68
{
"results": {
"drop": {
"em": 0.03313758389261745,
"em_stderr": 0.0018330841858875643,
"f1": 0.08634542785234882,
"f1_stderr": 0.0022136353860709133
},
"coqa": {
"f1": 0.4557083534540516,
"f1_stderr": 0.01876948425119881,
"em": 0.3298333333333334,
"em_stderr": 0.019473215823053027
},
"race": {
"acc": 0.3263157894736842,
"acc_stderr": 0.014510987877134932
}
},
"versions": {
"drop": 1,
"coqa": 1,
"race": 1
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=bigscience/bloom-1b1,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda:0",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment