Commit aea63162 authored by Leo Gao's avatar Leo Gao
Browse files

Fix SAT

parent d7c08d5b
import json import json
import random import random
import os import os
from lm_eval.base import Task, rf, mean from lm_eval.base import MultipleChoiceTask, rf, mean
from tqdm import auto as tqdm_lib from tqdm import auto as tqdm_lib
from . common import simple_accuracy_metric from . common import simple_accuracy_metric
import numpy as np import numpy as np
from ..utils import sh from ..utils import sh
class SATAnalogies(Task): class SATAnalogies(MultipleChoiceTask):
NEEDS_MANUAL_DL = True NEEDS_MANUAL_DL = True
def __init__(self): def __init__(self):
...@@ -61,7 +61,7 @@ class SATAnalogies(Task): ...@@ -61,7 +61,7 @@ class SATAnalogies(Task):
doc = { doc = {
'source': source, 'source': source,
'query': query.split(' ')[:2], 'query': query.split(' ')[:2],
'choices': [" {} is to {}".format(*c.split(' ')[:2]) for c in choices], 'choices': ["{} is to {}".format(*c.split(' ')[:2]) for c in choices],
'gold': ['a','b','c','d','e'].index(answer_key.strip()), 'gold': ['a','b','c','d','e'].index(answer_key.strip()),
} }
yield doc yield doc
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment