Skip to content

GitLab

Commit f275301a authored by haileyschoelkopf's avatar haileyschoelkopf Committed by Hailey Schoelkopf
Browse files

make tasks and models registered by decorators

parent e7c18e53
Hide whitespace changes
Inline Side-by-side
Showing with 432 additions and 317 deletions
lm_eval/api/model.py
View file @ f275301a
...@@ -2,6 +2,29 @@ import abc ...@@ -2,6 +2,29 @@ import abc
from lm_eval import utils from lm_eval import utils
MODEL_REGISTRY = {}
def register_model(name):
# TODO: should fairseq/elk be cited for this design pattern?
def decorate(cls):
assert (
issubclass(cls, LM)
), f"Model '{name}' ({cls.__name__}) must extend LM class"
assert (
name not in MODEL_REGISTRY
), f"Model named '{name}' conflicts with existing model!"
MODEL_REGISTRY[name] = cls
return cls
return decorate
def get_model(model_name):
return MODEL_REGISTRY[model_name]
class LM(abc.ABC): class LM(abc.ABC):
def __init__(self): def __init__(self):
......
lm_eval/api/task.py
View file @ f275301a
...@@ -9,6 +9,8 @@ import itertools ...@@ -9,6 +9,8 @@ import itertools
import datasets import datasets
import numpy as np import numpy as np
from typing import List, Union
from lm_eval.api import METRIC_REGISTRY, AGGREGATION_REGISTRY from lm_eval.api import METRIC_REGISTRY, AGGREGATION_REGISTRY
from lm_eval.api.instance import Instance from lm_eval.api.instance import Instance
from lm_eval.api.metrics import mean, weighted_perplexity, weighted_mean, bits_per_byte from lm_eval.api.metrics import mean, weighted_perplexity, weighted_mean, bits_per_byte
...@@ -31,7 +33,7 @@ class TaskConfig(dict): ...@@ -31,7 +33,7 @@ class TaskConfig(dict):
# TODO: add this as more jinja2 appended to start of jinja2 templates. Should allow users to set vars # TODO: add this as more jinja2 appended to start of jinja2 templates. Should allow users to set vars
# s.t. they can define e.g. {% set question = query %} to map dataset columns to "canonical" names in prompts. # s.t. they can define e.g. {% set question = query %} to map dataset columns to "canonical" names in prompts.
template_vars: str = None template_aliases: str = None
doc_to_text: str = None doc_to_text: str = None
doc_to_target: str = None doc_to_target: str = None
...@@ -609,3 +611,82 @@ class PerplexityTask(Task, abc.ABC): ...@@ -609,3 +611,82 @@ class PerplexityTask(Task, abc.ABC):
def count_words(cls, doc): def count_words(cls, doc):
"""Downstream tasks with custom word boundaries should override this!""" """Downstream tasks with custom word boundaries should override this!"""
return len(re.split(r"\s+", doc)) return len(re.split(r"\s+", doc))
# TODO: confirm we want this to go in this file
TASK_REGISTRY = {}
ALL_TASKS = []
def register_task(name):
def decorate(cls):
assert (
issubclass(cls, Task)
), f"Task '{name}' ({cls.__name__}) must extend Task class"
assert (
name not in TASK_REGISTRY
), f"Task named '{name}' conflicts with existing task!"
TASK_REGISTRY[name] = cls
ALL_TASKS = sorted(list(TASK_REGISTRY)) # TODO: this doesn't seem to import right.
return cls
return decorate
##### Task registry utils and setup.
# ALL_TASKS = sorted(list(TASK_REGISTRY))
def get_task(task_name):
try:
return TASK_REGISTRY[task_name]
except KeyError:
print("Available tasks:")
pprint(TASK_REGISTRY)
raise KeyError(f"Missing task {task_name}")
def get_task_name_from_object(task_object):
for name, class_ in TASK_REGISTRY.items():
if class_ is task_object:
return name
# this gives a mechanism for non-registered tasks to have a custom name anyways when reporting
return (
task_object.EVAL_HARNESS_NAME
if hasattr(task_object, "EVAL_HARNESS_NAME")
else type(task_object).__name__
)
def get_task_name_from_config(task_config):
return "configurable_{dataset_path}_{dataset_name}".format(**task_config)
def get_task_dict(task_name_list: List[Union[str, dict, Task]], num_fewshot=None): # TODO: pass num_fewshot and other cmdline overrides in a better way
task_name_dict = {
task_name: get_task(task_name)(config={"num_fewshot": num_fewshot if num_fewshot else 0, "task_name": task_name})
for task_name in task_name_list
if isinstance(task_name, str)
}
task_name_from_config_dict = {
get_task_name_from_config(task_config): ConfigurableTask(
config=task_config
)
for task_config in task_name_list
if isinstance(task_config, dict)
}
task_name_from_object_dict = {
get_task_name_from_object(task_object): task_object
for task_object in task_name_list
if isinstance(task_object, Task)
}
assert set(task_name_dict.keys()).isdisjoint(set(task_name_from_object_dict.keys()))
return {
**task_name_dict,
**task_name_from_config_dict,
**task_name_from_object_dict,
}
\ No newline at end of file
lm_eval/evaluator.py
View file @ f275301a
...@@ -58,14 +58,14 @@ def simple_evaluate( ...@@ -58,14 +58,14 @@ def simple_evaluate(
if isinstance(model, str): if isinstance(model, str):
if model_args is None: if model_args is None:
model_args = "" model_args = ""
lm = lm_eval.models.get_model(model).create_from_arg_string( lm = lm_eval.api.model.get_model(model).create_from_arg_string(
model_args, {"batch_size": batch_size, "device": device} model_args, {"batch_size": batch_size, "device": device}
) )
else: else:
assert isinstance(model, lm_eval.api.model.LM) assert isinstance(model, lm_eval.api.model.LM)
lm = model lm = model
task_dict = lm_eval.tasks.get_task_dict(tasks, num_fewshot=num_fewshot) task_dict = lm_eval.api.task.get_task_dict(tasks, num_fewshot=num_fewshot)
if check_integrity: if check_integrity:
run_task_tests(task_list=tasks) run_task_tests(task_list=tasks)
......
lm_eval/models/__init__.py
View file @ f275301a
from lm_eval.api.model import LM, MODEL_REGISTRY
from . import gpt2 from . import gpt2
from . import gpt3 from . import gpt3
from . import textsynth from . import textsynth
from . import dummy from . import dummy
MODEL_REGISTRY = { # MODEL_REGISTRY = {}
"hf-causal": gpt2.HFLM,
"openai": gpt3.GPT3LM,
"textsynth": textsynth.TextSynthLM,
"dummy": dummy.DummyLM,
}
# MODEL_REGISTRY = {
# "hf-causal": gpt2.HFLM,
# "openai": gpt3.GPT3LM,
# "textsynth": textsynth.TextSynthLM,
# "dummy": dummy.DummyLM,
# }
def get_model(model_name): # def get_model(model_name):
return MODEL_REGISTRY[model_name] # return MODEL_REGISTRY[model_name]
lm_eval/models/gpt2.py
View file @ f275301a
...@@ -6,9 +6,11 @@ from tqdm import tqdm ...@@ -6,9 +6,11 @@ from tqdm import tqdm
import torch.nn.functional as F import torch.nn.functional as F
from lm_eval import utils from lm_eval import utils
from lm_eval.api.model import LM from lm_eval.api.model import LM, register_model
# from lm_eval.models import register_model
@register_model("hf-causal")
class HFLM(LM): class HFLM(LM):
def __init__( def __init__(
self, self,
......
lm_eval/tasks/__init__.py
View file @ f275301a
...@@ -53,316 +53,316 @@ from . import gsm8k ...@@ -53,316 +53,316 @@ from . import gsm8k
# from . import toxigen # from . import toxigen
# from . import crowspairs # from . import crowspairs
######################################## # ########################################
# Translation tasks # # Translation tasks
######################################## # ########################################
# 6 total # # 6 total
gpt3_translation_benchmarks = { # gpt3_translation_benchmarks = {
"wmt14": ["en-fr", "fr-en"], # French # "wmt14": ["en-fr", "fr-en"], # French
"wmt16": ["en-ro", "ro-en", "de-en", "en-de"], # German, Romanian # "wmt16": ["en-ro", "ro-en", "de-en", "en-de"], # German, Romanian
} # }
# 28 total # # 28 total
selected_translation_benchmarks = { # selected_translation_benchmarks = {
**gpt3_translation_benchmarks, # **gpt3_translation_benchmarks,
"wmt20": sacrebleu.get_langpairs_for_testset("wmt20"), # "wmt20": sacrebleu.get_langpairs_for_testset("wmt20"),
"iwslt17": ["en-ar", "ar-en"], # Arabic # "iwslt17": ["en-ar", "ar-en"], # Arabic
} # }
# 319 total # # 319 total
all_translation_benchmarks = { # all_translation_benchmarks = {
ts: sacrebleu.get_langpairs_for_testset(ts) # ts: sacrebleu.get_langpairs_for_testset(ts)
for ts in sacrebleu.get_available_testsets() # for ts in sacrebleu.get_available_testsets()
} # }
######################################## # ########################################
# All tasks # # All tasks
######################################## # ########################################
TASK_REGISTRY = { # TASK_REGISTRY = {
# GLUE # # GLUE
# "cola": glue.CoLA, # # "cola": glue.CoLA,
# "mnli": glue.MNLI, # # "mnli": glue.MNLI,
# "mnli_mismatched": glue.MNLIMismatched, # # "mnli_mismatched": glue.MNLIMismatched,
# "mrpc": glue.MRPC, # # "mrpc": glue.MRPC,
# "rte": glue.RTE, # # "rte": glue.RTE,
# "qnli": glue.QNLI, # # "qnli": glue.QNLI,
# "qqp": glue.QQP, # # "qqp": glue.QQP,
# # "stsb": glue.STSB, # not implemented yet # # # "stsb": glue.STSB, # not implemented yet
# "sst": glue.SST, # # "sst": glue.SST,
# "wnli": glue.WNLI, # # "wnli": glue.WNLI,
# # SuperGLUE # # # SuperGLUE
# "boolq": superglue.BoolQ, # # "boolq": superglue.BoolQ,
# "cb": superglue.CommitmentBank, # # "cb": superglue.CommitmentBank,
# "copa": superglue.Copa, # # "copa": superglue.Copa,
# "multirc": superglue.MultiRC, # # "multirc": superglue.MultiRC,
# "record": superglue.ReCoRD, # # "record": superglue.ReCoRD,
# "wic": superglue.WordsInContext, # # "wic": superglue.WordsInContext,
# "wsc": superglue.SGWinogradSchemaChallenge, # # "wsc": superglue.SGWinogradSchemaChallenge,
# # Order by benchmark/genre? # # # Order by benchmark/genre?
# "coqa": coqa.CoQA, # # "coqa": coqa.CoQA,
# "drop": drop.DROP, # # "drop": drop.DROP,
"lambada_openai": lambada.LambadaOpenAI, # "lambada_openai": lambada.LambadaOpenAI,
"lambada_standard": lambada.LambadaStandard, # "lambada_standard": lambada.LambadaStandard,
# "lambada_openai_cloze": lambada_cloze.LambadaOpenAICloze, # # "lambada_openai_cloze": lambada_cloze.LambadaOpenAICloze,
# "lambada_standard_cloze": lambada_cloze.LambadaStandardCloze, # # "lambada_standard_cloze": lambada_cloze.LambadaStandardCloze,
# # multilingual lambada # # # multilingual lambada
# **lambada_multilingual.construct_tasks(), # # **lambada_multilingual.construct_tasks(),
"wikitext": wikitext.WikiText, # "wikitext": wikitext.WikiText,
# # "cbt-cn": cbt.CBTCN, # disabled pending context length fix # # # "cbt-cn": cbt.CBTCN, # disabled pending context length fix
# # "cbt-ne": cbt.CBTNE, # disabled pending context length fix # # # "cbt-ne": cbt.CBTNE, # disabled pending context length fix
# "piqa": piqa.PiQA, # # "piqa": piqa.PiQA,
# "prost": prost.PROST, # # "prost": prost.PROST,
# "mc_taco": mc_taco.MCTACO, # # "mc_taco": mc_taco.MCTACO,
# # Science related # # # Science related
# "pubmedqa": pubmedqa.Pubmed_QA, # # "pubmedqa": pubmedqa.Pubmed_QA,
# "sciq": sciq.SciQ, # # "sciq": sciq.SciQ,
# "qasper": qasper.QASPER, # # "qasper": qasper.QASPER,
# "qa4mre_2011": qa4mre.QA4MRE_2011, # # "qa4mre_2011": qa4mre.QA4MRE_2011,
# "qa4mre_2012": qa4mre.QA4MRE_2012, # # "qa4mre_2012": qa4mre.QA4MRE_2012,
# "qa4mre_2013": qa4mre.QA4MRE_2013, # # "qa4mre_2013": qa4mre.QA4MRE_2013,
# "triviaqa": triviaqa.TriviaQA, # # "triviaqa": triviaqa.TriviaQA,
"arc_easy": arc.ARCEasy, # "arc_easy": arc.ARCEasy,
"arc_challenge": arc.ARCChallenge, # "arc_challenge": arc.ARCChallenge,
# # "quac": quac.QuAC, # not implemented yet # # # "quac": quac.QuAC, # not implemented yet
# "logiqa": logiqa.LogiQA, # # "logiqa": logiqa.LogiQA,
# "hellaswag": hellaswag.HellaSwag, # # "hellaswag": hellaswag.HellaSwag,
# "swag": swag.SWAG, # # "swag": swag.SWAG,
# "openbookqa": openbookqa.OpenBookQA, # # "openbookqa": openbookqa.OpenBookQA,
# "squad2": squad.SQuAD2, # # "squad2": squad.SQuAD2,
# "race": race.RACE, # # "race": race.RACE,
# # "naturalqs": naturalqs.NaturalQs, # not implemented yet # # # "naturalqs": naturalqs.NaturalQs, # not implemented yet
# "headqa": headqa.HeadQAEsDeprecated, # for backwards compat - headqa used to default to es # # "headqa": headqa.HeadQAEsDeprecated, # for backwards compat - headqa used to default to es
# "headqa_es": headqa.HeadQAEs, # # "headqa_es": headqa.HeadQAEs,
# "headqa_en": headqa.HeadQAEn, # # "headqa_en": headqa.HeadQAEn,
# "mathqa": mathqa.MathQA, # # "mathqa": mathqa.MathQA,
# "webqs": webqs.WebQs, # # "webqs": webqs.WebQs,
# "wsc273": wsc273.WinogradSchemaChallenge273, # # "wsc273": wsc273.WinogradSchemaChallenge273,
# "winogrande": winogrande.Winogrande, # # "winogrande": winogrande.Winogrande,
# "anli_r1": anli.ANLIRound1, # # "anli_r1": anli.ANLIRound1,
# "anli_r2": anli.ANLIRound2, # # "anli_r2": anli.ANLIRound2,
# "anli_r3": anli.ANLIRound3, # # "anli_r3": anli.ANLIRound3,
# "ethics_cm": hendrycks_ethics.EthicsCM, # # "ethics_cm": hendrycks_ethics.EthicsCM,
# "ethics_deontology": hendrycks_ethics.EthicsDeontology, # # "ethics_deontology": hendrycks_ethics.EthicsDeontology,
# "ethics_justice": hendrycks_ethics.EthicsJustice, # # "ethics_justice": hendrycks_ethics.EthicsJustice,
# "ethics_utilitarianism_original": hendrycks_ethics.EthicsUtilitarianismOriginal, # # "ethics_utilitarianism_original": hendrycks_ethics.EthicsUtilitarianismOriginal,
# "ethics_utilitarianism": hendrycks_ethics.EthicsUtilitarianism, # # "ethics_utilitarianism": hendrycks_ethics.EthicsUtilitarianism,
# "ethics_virtue": hendrycks_ethics.EthicsVirtue, # # "ethics_virtue": hendrycks_ethics.EthicsVirtue,
# "truthfulqa_mc": truthfulqa.TruthfulQAMultipleChoice, # # "truthfulqa_mc": truthfulqa.TruthfulQAMultipleChoice,
# "truthfulqa_gen": truthfulqa.TruthfulQAGeneration, # # "truthfulqa_gen": truthfulqa.TruthfulQAGeneration,
# # dialogue # # # dialogue
# "mutual": mutual.MuTual, # # "mutual": mutual.MuTual,
# "mutual_plus": mutual.MuTualPlus, # # "mutual_plus": mutual.MuTualPlus,
# # math # # # math
# "math_algebra": hendrycks_math.MathAlgebra, # # "math_algebra": hendrycks_math.MathAlgebra,
# "math_counting_and_prob": hendrycks_math.MathCountingAndProbability, # # "math_counting_and_prob": hendrycks_math.MathCountingAndProbability,
# "math_geometry": hendrycks_math.MathGeometry, # # "math_geometry": hendrycks_math.MathGeometry,
# "math_intermediate_algebra": hendrycks_math.MathIntermediateAlgebra, # # "math_intermediate_algebra": hendrycks_math.MathIntermediateAlgebra,
# "math_num_theory": hendrycks_math.MathNumberTheory, # # "math_num_theory": hendrycks_math.MathNumberTheory,
# "math_prealgebra": hendrycks_math.MathPrealgebra, # # "math_prealgebra": hendrycks_math.MathPrealgebra,
# "math_precalc": hendrycks_math.MathPrecalculus, # # "math_precalc": hendrycks_math.MathPrecalculus,
# "math_asdiv": asdiv.Asdiv, # # "math_asdiv": asdiv.Asdiv,
"gsm8k": gsm8k.GradeSchoolMath8K, # "gsm8k": gsm8k.GradeSchoolMath8K,
# # arithmetic # # # arithmetic
# "arithmetic_2da": arithmetic.Arithmetic2DPlus, # # "arithmetic_2da": arithmetic.Arithmetic2DPlus,
# "arithmetic_2ds": arithmetic.Arithmetic2DMinus, # # "arithmetic_2ds": arithmetic.Arithmetic2DMinus,
# "arithmetic_3da": arithmetic.Arithmetic3DPlus, # # "arithmetic_3da": arithmetic.Arithmetic3DPlus,
# "arithmetic_3ds": arithmetic.Arithmetic3DMinus, # # "arithmetic_3ds": arithmetic.Arithmetic3DMinus,
# "arithmetic_4da": arithmetic.Arithmetic4DPlus, # # "arithmetic_4da": arithmetic.Arithmetic4DPlus,
# "arithmetic_4ds": arithmetic.Arithmetic4DMinus, # # "arithmetic_4ds": arithmetic.Arithmetic4DMinus,
# "arithmetic_5da": arithmetic.Arithmetic5DPlus, # # "arithmetic_5da": arithmetic.Arithmetic5DPlus,
# "arithmetic_5ds": arithmetic.Arithmetic5DMinus, # # "arithmetic_5ds": arithmetic.Arithmetic5DMinus,
# "arithmetic_2dm": arithmetic.Arithmetic2DMultiplication, # # "arithmetic_2dm": arithmetic.Arithmetic2DMultiplication,
# "arithmetic_1dc": arithmetic.Arithmetic1DComposite, # # "arithmetic_1dc": arithmetic.Arithmetic1DComposite,
# # TODO Perhaps make these groups of tasks # # # TODO Perhaps make these groups of tasks
# # e.g. anli, arithmetic, openai_translations, harness_translations # # # e.g. anli, arithmetic, openai_translations, harness_translations
# # hendrycksTest (57 tasks) # # # hendrycksTest (57 tasks)
# **hendrycks_test.create_all_tasks(), # # **hendrycks_test.create_all_tasks(),
# # e.g. wmt14-fr-en # # # e.g. wmt14-fr-en
# **translation.create_tasks_from_benchmarks(gpt3_translation_benchmarks), # # **translation.create_tasks_from_benchmarks(gpt3_translation_benchmarks),
# # chef's selection, mostly wmt20 # # # chef's selection, mostly wmt20
# **translation.create_tasks_from_benchmarks(selected_translation_benchmarks), # # **translation.create_tasks_from_benchmarks(selected_translation_benchmarks),
# # Word Scrambling and Manipulation Tasks # # # Word Scrambling and Manipulation Tasks
# "anagrams1": unscramble.Anagrams1, # # "anagrams1": unscramble.Anagrams1,
# "anagrams2": unscramble.Anagrams2, # # "anagrams2": unscramble.Anagrams2,
# "cycle_letters": unscramble.CycleLetters, # # "cycle_letters": unscramble.CycleLetters,
# "random_insertion": unscramble.RandomInsertion, # # "random_insertion": unscramble.RandomInsertion,
# "reversed_words": unscramble.ReversedWords, # # "reversed_words": unscramble.ReversedWords,
# # Pile # # # Pile
# "pile_arxiv": pile.PileArxiv, # # "pile_arxiv": pile.PileArxiv,
# "pile_books3": pile.PileBooks3, # # "pile_books3": pile.PileBooks3,
# "pile_bookcorpus2": pile.PileBookCorpus2, # # "pile_bookcorpus2": pile.PileBookCorpus2,
# "pile_dm-mathematics": pile.PileDmMathematics, # # "pile_dm-mathematics": pile.PileDmMathematics,
# "pile_enron": pile.PileEnron, # # "pile_enron": pile.PileEnron,
# "pile_europarl": pile.PileEuroparl, # # "pile_europarl": pile.PileEuroparl,
# "pile_freelaw": pile.PileFreeLaw, # # "pile_freelaw": pile.PileFreeLaw,
# "pile_github": pile.PileGithub, # # "pile_github": pile.PileGithub,
# "pile_gutenberg": pile.PileGutenberg, # # "pile_gutenberg": pile.PileGutenberg,
# "pile_hackernews": pile.PileHackernews, # # "pile_hackernews": pile.PileHackernews,
# "pile_nih-exporter": pile.PileNIHExporter, # # "pile_nih-exporter": pile.PileNIHExporter,
# "pile_opensubtitles": pile.PileOpenSubtitles, # # "pile_opensubtitles": pile.PileOpenSubtitles,
# "pile_openwebtext2": pile.PileOpenWebText2, # # "pile_openwebtext2": pile.PileOpenWebText2,
# "pile_philpapers": pile.PilePhilPapers, # # "pile_philpapers": pile.PilePhilPapers,
# "pile_pile-cc": pile.PilePileCc, # # "pile_pile-cc": pile.PilePileCc,
# "pile_pubmed-abstracts": pile.PilePubmedAbstracts, # # "pile_pubmed-abstracts": pile.PilePubmedAbstracts,
# "pile_pubmed-central": pile.PilePubmedCentral, # # "pile_pubmed-central": pile.PilePubmedCentral,
# "pile_stackexchange": pile.PileStackExchange, # # "pile_stackexchange": pile.PileStackExchange,
# "pile_uspto": pile.PileUspto, # # "pile_uspto": pile.PileUspto,
# "pile_ubuntu-irc": pile.PileUbuntuIrc, # # "pile_ubuntu-irc": pile.PileUbuntuIrc,
# "pile_wikipedia": pile.PileWikipedia, # # "pile_wikipedia": pile.PileWikipedia,
# "pile_youtubesubtitles": pile.PileYoutubeSubtitles, # # "pile_youtubesubtitles": pile.PileYoutubeSubtitles,
# # BLiMP # # # BLiMP
# "blimp_adjunct_island": blimp.BlimpAdjunctIsland, # # "blimp_adjunct_island": blimp.BlimpAdjunctIsland,
# "blimp_anaphor_gender_agreement": blimp.BlimpAnaphorGenderAgreement, # # "blimp_anaphor_gender_agreement": blimp.BlimpAnaphorGenderAgreement,
# "blimp_anaphor_number_agreement": blimp.BlimpAnaphorNumberAgreement, # # "blimp_anaphor_number_agreement": blimp.BlimpAnaphorNumberAgreement,
# "blimp_animate_subject_passive": blimp.BlimpAnimateSubjectPassive, # # "blimp_animate_subject_passive": blimp.BlimpAnimateSubjectPassive,
# "blimp_animate_subject_trans": blimp.BlimpAnimateSubjectTrans, # # "blimp_animate_subject_trans": blimp.BlimpAnimateSubjectTrans,
# "blimp_causative": blimp.BlimpCausative, # # "blimp_causative": blimp.BlimpCausative,
# "blimp_complex_NP_island": blimp.BlimpComplex_NPIsland, # # "blimp_complex_NP_island": blimp.BlimpComplex_NPIsland,
# "blimp_coordinate_structure_constraint_complex_left_branch": blimp.BlimpCoordinateStructureConstraintComplexLeftBranch, # # "blimp_coordinate_structure_constraint_complex_left_branch": blimp.BlimpCoordinateStructureConstraintComplexLeftBranch,
# "blimp_coordinate_structure_constraint_object_extraction": blimp.BlimpCoordinateStructureConstraintObjectExtraction, # # "blimp_coordinate_structure_constraint_object_extraction": blimp.BlimpCoordinateStructureConstraintObjectExtraction,
# "blimp_determiner_noun_agreement_1": blimp.BlimpDeterminerNounAgreement_1, # # "blimp_determiner_noun_agreement_1": blimp.BlimpDeterminerNounAgreement_1,
# "blimp_determiner_noun_agreement_2": blimp.BlimpDeterminerNounAgreement_2, # # "blimp_determiner_noun_agreement_2": blimp.BlimpDeterminerNounAgreement_2,
# "blimp_determiner_noun_agreement_irregular_1": blimp.BlimpDeterminerNounAgreementIrregular_1, # # "blimp_determiner_noun_agreement_irregular_1": blimp.BlimpDeterminerNounAgreementIrregular_1,
# "blimp_determiner_noun_agreement_irregular_2": blimp.BlimpDeterminerNounAgreementIrregular_2, # # "blimp_determiner_noun_agreement_irregular_2": blimp.BlimpDeterminerNounAgreementIrregular_2,
# "blimp_determiner_noun_agreement_with_adj_2": blimp.BlimpDeterminerNounAgreementWithAdj_2, # # "blimp_determiner_noun_agreement_with_adj_2": blimp.BlimpDeterminerNounAgreementWithAdj_2,
# "blimp_determiner_noun_agreement_with_adj_irregular_1": blimp.BlimpDeterminerNounAgreementWithAdjIrregular_1, # # "blimp_determiner_noun_agreement_with_adj_irregular_1": blimp.BlimpDeterminerNounAgreementWithAdjIrregular_1,
# "blimp_determiner_noun_agreement_with_adj_irregular_2": blimp.BlimpDeterminerNounAgreementWithAdjIrregular_2, # # "blimp_determiner_noun_agreement_with_adj_irregular_2": blimp.BlimpDeterminerNounAgreementWithAdjIrregular_2,
# "blimp_determiner_noun_agreement_with_adjective_1": blimp.BlimpDeterminerNounAgreementWithAdjective_1, # # "blimp_determiner_noun_agreement_with_adjective_1": blimp.BlimpDeterminerNounAgreementWithAdjective_1,
# "blimp_distractor_agreement_relational_noun": blimp.BlimpDistractorAgreementRelationalNoun, # # "blimp_distractor_agreement_relational_noun": blimp.BlimpDistractorAgreementRelationalNoun,
# "blimp_distractor_agreement_relative_clause": blimp.BlimpDistractorAgreementRelativeClause, # # "blimp_distractor_agreement_relative_clause": blimp.BlimpDistractorAgreementRelativeClause,
# "blimp_drop_argument": blimp.BlimpDropArgument, # # "blimp_drop_argument": blimp.BlimpDropArgument,
# "blimp_ellipsis_n_bar_1": blimp.BlimpEllipsisNBar_1, # # "blimp_ellipsis_n_bar_1": blimp.BlimpEllipsisNBar_1,
# "blimp_ellipsis_n_bar_2": blimp.BlimpEllipsisNBar_2, # # "blimp_ellipsis_n_bar_2": blimp.BlimpEllipsisNBar_2,
# "blimp_existential_there_object_raising": blimp.BlimpExistentialThereObjectRaising, # # "blimp_existential_there_object_raising": blimp.BlimpExistentialThereObjectRaising,
# "blimp_existential_there_quantifiers_1": blimp.BlimpExistentialThereQuantifiers_1, # # "blimp_existential_there_quantifiers_1": blimp.BlimpExistentialThereQuantifiers_1,
# "blimp_existential_there_quantifiers_2": blimp.BlimpExistentialThereQuantifiers_2, # # "blimp_existential_there_quantifiers_2": blimp.BlimpExistentialThereQuantifiers_2,
# "blimp_existential_there_subject_raising": blimp.BlimpExistentialThereSubjectRaising, # # "blimp_existential_there_subject_raising": blimp.BlimpExistentialThereSubjectRaising,
# "blimp_expletive_it_object_raising": blimp.BlimpExpletiveItObjectRaising, # # "blimp_expletive_it_object_raising": blimp.BlimpExpletiveItObjectRaising,
# "blimp_inchoative": blimp.BlimpInchoative, # # "blimp_inchoative": blimp.BlimpInchoative,
# "blimp_intransitive": blimp.BlimpIntransitive, # # "blimp_intransitive": blimp.BlimpIntransitive,
# "blimp_irregular_past_participle_adjectives": blimp.BlimpIrregularPastParticipleAdjectives, # # "blimp_irregular_past_participle_adjectives": blimp.BlimpIrregularPastParticipleAdjectives,
# "blimp_irregular_past_participle_verbs": blimp.BlimpIrregularPastParticipleVerbs, # # "blimp_irregular_past_participle_verbs": blimp.BlimpIrregularPastParticipleVerbs,
# "blimp_irregular_plural_subject_verb_agreement_1": blimp.BlimpIrregularPluralSubjectVerbAgreement_1, # # "blimp_irregular_plural_subject_verb_agreement_1": blimp.BlimpIrregularPluralSubjectVerbAgreement_1,
# "blimp_irregular_plural_subject_verb_agreement_2": blimp.BlimpIrregularPluralSubjectVerbAgreement_2, # # "blimp_irregular_plural_subject_verb_agreement_2": blimp.BlimpIrregularPluralSubjectVerbAgreement_2,
# "blimp_left_branch_island_echo_question": blimp.BlimpLeftBranchIslandEchoQuestion, # # "blimp_left_branch_island_echo_question": blimp.BlimpLeftBranchIslandEchoQuestion,
# "blimp_left_branch_island_simple_question": blimp.BlimpLeftBranchIslandSimpleQuestion, # # "blimp_left_branch_island_simple_question": blimp.BlimpLeftBranchIslandSimpleQuestion,
# "blimp_matrix_question_npi_licensor_present": blimp.BlimpMatrixQuestionNpiLicensorPresent, # # "blimp_matrix_question_npi_licensor_present": blimp.BlimpMatrixQuestionNpiLicensorPresent,
# "blimp_npi_present_1": blimp.BlimpNpiPresent_1, # # "blimp_npi_present_1": blimp.BlimpNpiPresent_1,
# "blimp_npi_present_2": blimp.BlimpNpiPresent_2, # # "blimp_npi_present_2": blimp.BlimpNpiPresent_2,
# "blimp_only_npi_licensor_present": blimp.BlimpOnlyNpiLicensorPresent, # # "blimp_only_npi_licensor_present": blimp.BlimpOnlyNpiLicensorPresent,
# "blimp_only_npi_scope": blimp.BlimpOnlyNpiScope, # # "blimp_only_npi_scope": blimp.BlimpOnlyNpiScope,
# "blimp_passive_1": blimp.BlimpPassive_1, # # "blimp_passive_1": blimp.BlimpPassive_1,
# "blimp_passive_2": blimp.BlimpPassive_2, # # "blimp_passive_2": blimp.BlimpPassive_2,
# "blimp_principle_A_c_command": blimp.BlimpPrinciple_ACCommand, # # "blimp_principle_A_c_command": blimp.BlimpPrinciple_ACCommand,
# "blimp_principle_A_case_1": blimp.BlimpPrinciple_ACase_1, # # "blimp_principle_A_case_1": blimp.BlimpPrinciple_ACase_1,
# "blimp_principle_A_case_2": blimp.BlimpPrinciple_ACase_2, # # "blimp_principle_A_case_2": blimp.BlimpPrinciple_ACase_2,
# "blimp_principle_A_domain_1": blimp.BlimpPrinciple_ADomain_1, # # "blimp_principle_A_domain_1": blimp.BlimpPrinciple_ADomain_1,
# "blimp_principle_A_domain_2": blimp.BlimpPrinciple_ADomain_2, # # "blimp_principle_A_domain_2": blimp.BlimpPrinciple_ADomain_2,
# "blimp_principle_A_domain_3": blimp.BlimpPrinciple_ADomain_3, # # "blimp_principle_A_domain_3": blimp.BlimpPrinciple_ADomain_3,
# "blimp_principle_A_reconstruction": blimp.BlimpPrinciple_AReconstruction, # # "blimp_principle_A_reconstruction": blimp.BlimpPrinciple_AReconstruction,
# "blimp_regular_plural_subject_verb_agreement_1": blimp.BlimpRegularPluralSubjectVerbAgreement_1, # # "blimp_regular_plural_subject_verb_agreement_1": blimp.BlimpRegularPluralSubjectVerbAgreement_1,
# "blimp_regular_plural_subject_verb_agreement_2": blimp.BlimpRegularPluralSubjectVerbAgreement_2, # # "blimp_regular_plural_subject_verb_agreement_2": blimp.BlimpRegularPluralSubjectVerbAgreement_2,
# "blimp_sentential_negation_npi_licensor_present": blimp.BlimpSententialNegationNpiLicensorPresent, # # "blimp_sentential_negation_npi_licensor_present": blimp.BlimpSententialNegationNpiLicensorPresent,
# "blimp_sentential_negation_npi_scope": blimp.BlimpSententialNegationNpiScope, # # "blimp_sentential_negation_npi_scope": blimp.BlimpSententialNegationNpiScope,
# "blimp_sentential_subject_island": blimp.BlimpSententialSubjectIsland, # # "blimp_sentential_subject_island": blimp.BlimpSententialSubjectIsland,
# "blimp_superlative_quantifiers_1": blimp.BlimpSuperlativeQuantifiers_1, # # "blimp_superlative_quantifiers_1": blimp.BlimpSuperlativeQuantifiers_1,
# "blimp_superlative_quantifiers_2": blimp.BlimpSuperlativeQuantifiers_2, # # "blimp_superlative_quantifiers_2": blimp.BlimpSuperlativeQuantifiers_2,
# "blimp_tough_vs_raising_1": blimp.BlimpToughVsRaising_1, # # "blimp_tough_vs_raising_1": blimp.BlimpToughVsRaising_1,
# "blimp_tough_vs_raising_2": blimp.BlimpToughVsRaising_2, # # "blimp_tough_vs_raising_2": blimp.BlimpToughVsRaising_2,
# "blimp_transitive": blimp.BlimpTransitive, # # "blimp_transitive": blimp.BlimpTransitive,
# "blimp_wh_island": blimp.BlimpWhIsland, # # "blimp_wh_island": blimp.BlimpWhIsland,
# "blimp_wh_questions_object_gap": blimp.BlimpWhQuestionsObjectGap, # # "blimp_wh_questions_object_gap": blimp.BlimpWhQuestionsObjectGap,
# "blimp_wh_questions_subject_gap": blimp.BlimpWhQuestionsSubjectGap, # # "blimp_wh_questions_subject_gap": blimp.BlimpWhQuestionsSubjectGap,
# "blimp_wh_questions_subject_gap_long_distance": blimp.BlimpWhQuestionsSubjectGapLongDistance, # # "blimp_wh_questions_subject_gap_long_distance": blimp.BlimpWhQuestionsSubjectGapLongDistance,
# "blimp_wh_vs_that_no_gap": blimp.BlimpWhVsThatNoGap, # # "blimp_wh_vs_that_no_gap": blimp.BlimpWhVsThatNoGap,
# "blimp_wh_vs_that_no_gap_long_distance": blimp.BlimpWhVsThatNoGapLongDistance, # # "blimp_wh_vs_that_no_gap_long_distance": blimp.BlimpWhVsThatNoGapLongDistance,
# "blimp_wh_vs_that_with_gap": blimp.BlimpWhVsThatWithGap, # # "blimp_wh_vs_that_with_gap": blimp.BlimpWhVsThatWithGap,
# "blimp_wh_vs_that_with_gap_long_distance": blimp.BlimpWhVsThatWithGapLongDistance, # # "blimp_wh_vs_that_with_gap_long_distance": blimp.BlimpWhVsThatWithGapLongDistance,
# "toxigen": toxigen.ToxiGen, # # "toxigen": toxigen.ToxiGen,
# "crows_pairs_english": crowspairs.CrowsPairsEnglish, # # "crows_pairs_english": crowspairs.CrowsPairsEnglish,
# "crows_pairs_english_race_color": crowspairs.CrowsPairsEnglishRaceColor, # # "crows_pairs_english_race_color": crowspairs.CrowsPairsEnglishRaceColor,
# "crows_pairs_english_socioeconomic": crowspairs.CrowsPairsEnglishSocioeconomic, # # "crows_pairs_english_socioeconomic": crowspairs.CrowsPairsEnglishSocioeconomic,
# "crows_pairs_english_gender": crowspairs.CrowsPairsEnglishGender, # # "crows_pairs_english_gender": crowspairs.CrowsPairsEnglishGender,
# "crows_pairs_english_age": crowspairs.CrowsPairsEnglishAge, # # "crows_pairs_english_age": crowspairs.CrowsPairsEnglishAge,
# "crows_pairs_english_religion": crowspairs.CrowsPairsEnglishReligion, # # "crows_pairs_english_religion": crowspairs.CrowsPairsEnglishReligion,
# "crows_pairs_english_disability": crowspairs.CrowsPairsEnglishDisability, # # "crows_pairs_english_disability": crowspairs.CrowsPairsEnglishDisability,
# "crows_pairs_english_sexual_orientation": crowspairs.CrowsPairsEnglishSexualOrientation, # # "crows_pairs_english_sexual_orientation": crowspairs.CrowsPairsEnglishSexualOrientation,
# "crows_pairs_english_nationality": crowspairs.CrowsPairsEnglishNationality, # # "crows_pairs_english_nationality": crowspairs.CrowsPairsEnglishNationality,
# "crows_pairs_english_physical_appearance": crowspairs.CrowsPairsEnglishPhysicalAppearance, # # "crows_pairs_english_physical_appearance": crowspairs.CrowsPairsEnglishPhysicalAppearance,
# "crows_pairs_english_autre": crowspairs.CrowsPairsEnglishAutre, # # "crows_pairs_english_autre": crowspairs.CrowsPairsEnglishAutre,
# "crows_pairs_french": crowspairs.CrowsPairsFrench, # # "crows_pairs_french": crowspairs.CrowsPairsFrench,
# "crows_pairs_french_race_color": crowspairs.CrowsPairsFrenchRaceColor, # # "crows_pairs_french_race_color": crowspairs.CrowsPairsFrenchRaceColor,
# "crows_pairs_french_socioeconomic": crowspairs.CrowsPairsFrenchSocioeconomic, # # "crows_pairs_french_socioeconomic": crowspairs.CrowsPairsFrenchSocioeconomic,
# "crows_pairs_french_gender": crowspairs.CrowsPairsFrenchGender, # # "crows_pairs_french_gender": crowspairs.CrowsPairsFrenchGender,
# "crows_pairs_french_age": crowspairs.CrowsPairsFrenchAge, # # "crows_pairs_french_age": crowspairs.CrowsPairsFrenchAge,
# "crows_pairs_french_religion": crowspairs.CrowsPairsFrenchReligion, # # "crows_pairs_french_religion": crowspairs.CrowsPairsFrenchReligion,
# "crows_pairs_french_disability": crowspairs.CrowsPairsFrenchDisability, # # "crows_pairs_french_disability": crowspairs.CrowsPairsFrenchDisability,
# "crows_pairs_french_sexual_orientation": crowspairs.CrowsPairsFrenchSexualOrientation, # # "crows_pairs_french_sexual_orientation": crowspairs.CrowsPairsFrenchSexualOrientation,
# "crows_pairs_french_nationality": crowspairs.CrowsPairsFrenchNationality, # # "crows_pairs_french_nationality": crowspairs.CrowsPairsFrenchNationality,
# "crows_pairs_french_physical_appearance": crowspairs.CrowsPairsFrenchPhysicalAppearance, # # "crows_pairs_french_physical_appearance": crowspairs.CrowsPairsFrenchPhysicalAppearance,
# "crows_pairs_french_autre": crowspairs.CrowsPairsFrenchAutre, # # "crows_pairs_french_autre": crowspairs.CrowsPairsFrenchAutre,
# Requires manual download of data. # # Requires manual download of data.
# "storycloze_2016": storycloze.StoryCloze2016, # # "storycloze_2016": storycloze.StoryCloze2016,
# "storycloze_2018": storycloze.StoryCloze2018, # # "storycloze_2018": storycloze.StoryCloze2018,
# "sat": sat.SATAnalogies, # # "sat": sat.SATAnalogies,
} # }
ALL_TASKS = sorted(list(TASK_REGISTRY)) # ALL_TASKS = sorted(list(TASK_REGISTRY))
def get_task(task_name): # def get_task(task_name):
try: # try:
return TASK_REGISTRY[task_name] # return TASK_REGISTRY[task_name]
except KeyError: # except KeyError:
print("Available tasks:") # print("Available tasks:")
pprint(TASK_REGISTRY) # pprint(TASK_REGISTRY)
raise KeyError(f"Missing task {task_name}") # raise KeyError(f"Missing task {task_name}")
def get_task_name_from_object(task_object): # def get_task_name_from_object(task_object):
for name, class_ in TASK_REGISTRY.items(): # for name, class_ in TASK_REGISTRY.items():
if class_ is task_object: # if class_ is task_object:
return name # return name
# this gives a mechanism for non-registered tasks to have a custom name anyways when reporting # # this gives a mechanism for non-registered tasks to have a custom name anyways when reporting
return ( # return (
task_object.EVAL_HARNESS_NAME # task_object.EVAL_HARNESS_NAME
if hasattr(task_object, "EVAL_HARNESS_NAME") # if hasattr(task_object, "EVAL_HARNESS_NAME")
else type(task_object).__name__ # else type(task_object).__name__
) # )
def get_task_name_from_config(task_config): # def get_task_name_from_config(task_config):
return "configurable_{dataset_path}_{dataset_name}".format(**task_config) # return "configurable_{dataset_path}_{dataset_name}".format(**task_config)
def get_task_dict(task_name_list: List[Union[str, dict, api.task.Task]], num_fewshot=None): # TODO: pass num_fewshot and other cmdline overrides in a better way # def get_task_dict(task_name_list: List[Union[str, dict, api.task.Task]], num_fewshot=None): # TODO: pass num_fewshot and other cmdline overrides in a better way
task_name_dict = { # task_name_dict = {
task_name: get_task(task_name)(config={"num_fewshot": num_fewshot if num_fewshot else 0, "task_name": task_name}) # task_name: get_task(task_name)(config={"num_fewshot": num_fewshot if num_fewshot else 0, "task_name": task_name})
for task_name in task_name_list # for task_name in task_name_list
if isinstance(task_name, str) # if isinstance(task_name, str)
} # }
task_name_from_config_dict = { # task_name_from_config_dict = {
get_task_name_from_config(task_config): api.task.ConfigurableTask( # get_task_name_from_config(task_config): api.task.ConfigurableTask(
config=task_config # config=task_config
) # )
for task_config in task_name_list # for task_config in task_name_list
if isinstance(task_config, dict) # if isinstance(task_config, dict)
} # }
task_name_from_object_dict = { # task_name_from_object_dict = {
get_task_name_from_object(task_object): task_object # get_task_name_from_object(task_object): task_object
for task_object in task_name_list # for task_object in task_name_list
if isinstance(task_object, api.task.Task) # if isinstance(task_object, api.task.Task)
} # }
assert set(task_name_dict.keys()).isdisjoint(set(task_name_from_object_dict.keys())) # assert set(task_name_dict.keys()).isdisjoint(set(task_name_from_object_dict.keys()))
return { # return {
**task_name_dict, # **task_name_dict,
**task_name_from_config_dict, # **task_name_from_config_dict,
**task_name_from_object_dict, # **task_name_from_object_dict,
} # }
\ No newline at end of file \ No newline at end of file
lm_eval/tasks/arc.py
View file @ f275301a
...@@ -12,7 +12,7 @@ a co-occurrence method fail to answer correctly) and an Easy Set of 5,197 questi ...@@ -12,7 +12,7 @@ a co-occurrence method fail to answer correctly) and an Easy Set of 5,197 questi
Homepage: https://allenai.org/data/arc Homepage: https://allenai.org/data/arc
""" """
from lm_eval.api.task import MultipleChoiceTask from lm_eval.api.task import MultipleChoiceTask, register_task
from lm_eval.prompts import get_prompt from lm_eval.prompts import get_prompt
from lm_eval import utils from lm_eval import utils
...@@ -28,7 +28,7 @@ _CITATION = """ ...@@ -28,7 +28,7 @@ _CITATION = """
} }
""" """
@register_task("arc_easy")
class ARCEasy(MultipleChoiceTask): class ARCEasy(MultipleChoiceTask):
VERSION = "2.0" VERSION = "2.0"
DATASET_PATH = "ai2_arc" DATASET_PATH = "ai2_arc"
...@@ -80,6 +80,7 @@ class ARCEasy(MultipleChoiceTask): ...@@ -80,6 +80,7 @@ class ARCEasy(MultipleChoiceTask):
return doc["query"] return doc["query"]
@register_task("arc_challenge")
class ARCChallenge(ARCEasy): class ARCChallenge(ARCEasy):
DATASET_PATH = "ai2_arc" DATASET_PATH = "ai2_arc"
DATASET_NAME = "ARC-Challenge" DATASET_NAME = "ARC-Challenge"
lm_eval/tasks/gsm8k.py
View file @ f275301a
...@@ -17,7 +17,7 @@ model's sample/generation function. ...@@ -17,7 +17,7 @@ model's sample/generation function.
Homepage: https://github.com/openai/grade-school-math Homepage: https://github.com/openai/grade-school-math
""" """
import re import re
from lm_eval.api.task import Task from lm_eval.api.task import Task, register_task
from lm_eval.api.instance import Instance from lm_eval.api.instance import Instance
from lm_eval.api.metrics import mean from lm_eval.api.metrics import mean
...@@ -41,6 +41,7 @@ ANS_RE = re.compile(r"#### (\-?[0-9\.\,]+)") ...@@ -41,6 +41,7 @@ ANS_RE = re.compile(r"#### (\-?[0-9\.\,]+)")
INVALID_ANS = "[invalid]" INVALID_ANS = "[invalid]"
@register_task("gsm8k")
class GradeSchoolMath8K(Task): class GradeSchoolMath8K(Task):
VERSION = 0 VERSION = 0
DATASET_PATH = "gsm8k" DATASET_PATH = "gsm8k"
......
lm_eval/tasks/lambada.py
View file @ f275301a
...@@ -12,7 +12,7 @@ in the broader discourse. ...@@ -12,7 +12,7 @@ in the broader discourse.
Homepage: https://zenodo.org/record/2630551#.X4Xzn5NKjUI Homepage: https://zenodo.org/record/2630551#.X4Xzn5NKjUI
""" """
from lm_eval.api.task import Task from lm_eval.api.task import Task, register_task
from lm_eval.api.instance import Instance from lm_eval.api.instance import Instance
from lm_eval.api.metrics import mean, perplexity from lm_eval.api.metrics import mean, perplexity
...@@ -75,6 +75,7 @@ class LambadaBase(Task): ...@@ -75,6 +75,7 @@ class LambadaBase(Task):
return {"ppl": False, "acc": True} return {"ppl": False, "acc": True}
@register_task("lambada_standard")
class LambadaStandard(LambadaBase): class LambadaStandard(LambadaBase):
"""The LAMBADA task using the standard original LAMBADA dataset.""" """The LAMBADA task using the standard original LAMBADA dataset."""
...@@ -90,7 +91,7 @@ class LambadaStandard(LambadaBase): ...@@ -90,7 +91,7 @@ class LambadaStandard(LambadaBase):
def has_test_docs(self): def has_test_docs(self):
return True return True
@register_task("lambada_openai")
class LambadaOpenAI(LambadaBase): class LambadaOpenAI(LambadaBase):
"""The LAMBADA task using the LAMBADA OpenAI dataset, a modified version of the """The LAMBADA task using the LAMBADA OpenAI dataset, a modified version of the
original LAMBADA dataset created by OpenAI for evaluating their GPT-2 model. original LAMBADA dataset created by OpenAI for evaluating their GPT-2 model.
......
lm_eval/tasks/wikitext.py
View file @ f275301a
...@@ -10,7 +10,7 @@ NOTE: This `Task` is based on WikiText-2. ...@@ -10,7 +10,7 @@ NOTE: This `Task` is based on WikiText-2.
Homepage: https://www.salesforce.com/products/einstein/ai-research/the-wikitext-dependency-language-modeling-dataset/ Homepage: https://www.salesforce.com/products/einstein/ai-research/the-wikitext-dependency-language-modeling-dataset/
""" """
import re import re
from lm_eval.api.task import PerplexityTask from lm_eval.api.task import PerplexityTask, register_task
_CITATION = """ _CITATION = """
...@@ -58,7 +58,7 @@ def wikitext_detokenizer(string): ...@@ -58,7 +58,7 @@ def wikitext_detokenizer(string):
return string return string
@register_task("wikitext")
class WikiText(PerplexityTask): class WikiText(PerplexityTask):
VERSION = "2.0" VERSION = "2.0"
DATASET_PATH = "EleutherAI/wikitext_document_level" DATASET_PATH = "EleutherAI/wikitext_document_level"
......
main.py
View file @ f275301a
...@@ -5,14 +5,17 @@ import fnmatch ...@@ -5,14 +5,17 @@ import fnmatch
import yaml import yaml
from lm_eval import tasks, evaluator from lm_eval import tasks, evaluator
from lm_eval.api.task import ConfigurableTask # import lm_eval.api.task
from lm_eval.api.task import ConfigurableTask, TASK_REGISTRY
logging.getLogger("openai").setLevel(logging.WARNING) logging.getLogger("openai").setLevel(logging.WARNING)
ALL_TASKS = sorted(list(TASK_REGISTRY))
class MultiChoice: class MultiChoice:
def __init__(self, choices): def __init__(self, choices):
self.choices = choices self.choices = choices
print(f"{ALL_TASKS} is this")
# Simple wildcard support (linux filename patterns) # Simple wildcard support (linux filename patterns)
def __contains__(self, values): def __contains__(self, values):
...@@ -31,7 +34,7 @@ def parse_args(): ...@@ -31,7 +34,7 @@ def parse_args():
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument("--model", required=True) parser.add_argument("--model", required=True)
parser.add_argument("--model_args", default="") parser.add_argument("--model_args", default="")
parser.add_argument("--tasks", default=None, choices=MultiChoice(tasks.ALL_TASKS)) parser.add_argument("--tasks", default=None, choices=MultiChoice(ALL_TASKS))
parser.add_argument("--config", default=None) parser.add_argument("--config", default=None)
parser.add_argument("--provide_description", action="store_true") parser.add_argument("--provide_description", action="store_true")
parser.add_argument("--num_fewshot", type=int, default=0) parser.add_argument("--num_fewshot", type=int, default=0)
...@@ -80,9 +83,9 @@ def main(): ...@@ -80,9 +83,9 @@ def main():
task_names.append(config) task_names.append(config)
else: else:
task_names = tasks.ALL_TASKS task_names = ALL_TASKS
else: else:
task_names = pattern_match(args.tasks.split(","), tasks.ALL_TASKS) task_names = pattern_match(args.tasks.split(","), ALL_TASKS)
print(f"Selected Tasks: {task_names}") print(f"Selected Tasks: {task_names}")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment