Skip to content

GitLab

Commit 5855f482 authored by thomasw21's avatar thomasw21
Browse files

Allow users to specify en headqa or es

parent 134150a2
Hide whitespace changes
Inline Side-by-side
Showing with 11 additions and 4 deletions
README.md
View file @ 5855f482
...@@ -128,7 +128,8 @@ To implement a new task in eval harness, see [this guide](https://github.com/Ele ...@@ -128,7 +128,8 @@ To implement a new task in eval harness, see [this guide](https://github.com/Ele
|openbookqa |✓ |✓ |✓ | 500|acc, acc_norm | |openbookqa |✓ |✓ |✓ | 500|acc, acc_norm |
|squad2 |✓ |✓ | | 11873|exact, f1, HasAns_exact, HasAns_f1, NoAns_exact, NoAns_f1, best_exact, best_f1| |squad2 |✓ |✓ | | 11873|exact, f1, HasAns_exact, HasAns_f1, NoAns_exact, NoAns_f1, best_exact, best_f1|
|race |✓ |✓ |✓ | 1045|acc | |race |✓ |✓ |✓ | 1045|acc |
|headqa |✓ |✓ |✓ | 2742|acc, acc_norm | |headqa_es |✓ |✓ |✓ | 2742|acc, acc_norm |
|headqa_en |✓ |✓ |✓ | 2742|acc, acc_norm |
|mathqa |✓ |✓ |✓ | 2985|acc, acc_norm | |mathqa |✓ |✓ |✓ | 2985|acc, acc_norm |
|webqs |✓ | |✓ | 2032|acc | |webqs |✓ | |✓ | 2032|acc |
|wsc273 | | |✓ | 273|acc | |wsc273 | | |✓ | 273|acc |
......
lm_eval/tasks/__init__.py
View file @ 5855f482
...@@ -132,7 +132,8 @@ TASK_REGISTRY = { ...@@ -132,7 +132,8 @@ TASK_REGISTRY = {
"squad2": squad.SQuAD2, "squad2": squad.SQuAD2,
"race": race.RACE, "race": race.RACE,
# "naturalqs": naturalqs.NaturalQs, # not implemented yet # "naturalqs": naturalqs.NaturalQs, # not implemented yet
"headqa": headqa.HeadQA, "headqa_es": headqa.HeadQAEs,
"headqa_en": headqa.HeadQAEn,
"mathqa": mathqa.MathQA, "mathqa": mathqa.MathQA,
"webqs": webqs.WebQs, "webqs": webqs.WebQs,
"wsc273": wsc273.WinogradSchemaChallenge273, "wsc273": wsc273.WinogradSchemaChallenge273,
......
lm_eval/tasks/headqa.py
View file @ 5855f482
...@@ -2,10 +2,9 @@ from . common import HFTask ...@@ -2,10 +2,9 @@ from . common import HFTask
from lm_eval.base import MultipleChoiceTask from lm_eval.base import MultipleChoiceTask
class HeadQA(HFTask, MultipleChoiceTask): class HeadQABase(HFTask, MultipleChoiceTask):
VERSION = 0 VERSION = 0
DATASET_PATH = "head_qa" DATASET_PATH = "head_qa"
DATASET_NAME = None
def has_training_docs(self): def has_training_docs(self):
return True return True
...@@ -31,3 +30,9 @@ class HeadQA(HFTask, MultipleChoiceTask): ...@@ -31,3 +30,9 @@ class HeadQA(HFTask, MultipleChoiceTask):
def doc_to_text(self, doc): def doc_to_text(self, doc):
return doc["query"] return doc["query"]
class HeadQAEn(HeadQABase):
DATASET_NAME = "en"
class HeadQAEs(HeadQABase):
DATASET_NAME = "es"
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment