Commit aea63162 authored by Leo Gao's avatar Leo Gao
Browse files

Fix SAT

parent d7c08d5b
import json
import random
import os
from lm_eval.base import Task, rf, mean
from lm_eval.base import MultipleChoiceTask, rf, mean
from tqdm import auto as tqdm_lib
from . common import simple_accuracy_metric
import numpy as np
from ..utils import sh
class SATAnalogies(Task):
class SATAnalogies(MultipleChoiceTask):
NEEDS_MANUAL_DL = True
def __init__(self):
......@@ -61,7 +61,7 @@ class SATAnalogies(Task):
doc = {
'source': source,
'query': query.split(' ')[:2],
'choices': [" {} is to {}".format(*c.split(' ')[:2]) for c in choices],
'choices': ["{} is to {}".format(*c.split(' ')[:2]) for c in choices],
'gold': ['a','b','c','d','e'].index(answer_key.strip()),
}
yield doc
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment