Commit dea15510 authored by Eric Tang's avatar Eric Tang
Browse files

Fixes to process_results string parsing

parent bbac75cf
...@@ -37,6 +37,7 @@ class Math(Task): ...@@ -37,6 +37,7 @@ class Math(Task):
for doc in self._testing_docs: for doc in self._testing_docs:
doc["answer"] = self.remove_boxed(self.last_boxed_only_string(doc["solution"])) doc["answer"] = self.remove_boxed(self.last_boxed_only_string(doc["solution"]))
self._testing_docs = self._testing_docs[200:300]
def has_training_docs(self): def has_training_docs(self):
return True return True
...@@ -57,7 +58,7 @@ class Math(Task): ...@@ -57,7 +58,7 @@ class Math(Task):
return self._testing_docs return self._testing_docs
def fewshot_examples(self, k): def fewshot_examples(self, k):
assert k <= 7, "There are only 7 possible shots for this task." assert k <= 8, "There are only 8 possible shots for this task."
prompts = [ prompts = [
{"problem": "What is $\left(\\frac{7}{8}\\right)^3 \cdot \left(\\frac{7}{8}\\right)^{-3}$?", "answer": "$1$"}, {"problem": "What is $\left(\\frac{7}{8}\\right)^3 \cdot \left(\\frac{7}{8}\\right)^{-3}$?", "answer": "$1$"},
{"problem": "In how many ways can 4 books be selected from a shelf of 6 books if the order in which the books are selected does not matter?", "answer": "$15$"}, {"problem": "In how many ways can 4 books be selected from a shelf of 6 books if the order in which the books are selected does not matter?", "answer": "$15$"},
...@@ -86,8 +87,12 @@ class Math(Task): ...@@ -86,8 +87,12 @@ class Math(Task):
def process_results(self, doc, results): def process_results(self, doc, results):
retval = 0 retval = 0
print(results)
indices = [pos for pos, char in enumerate(results[0]) if char == "$"] indices = [pos for pos, char in enumerate(results[0]) if char == "$"]
answer = results[0][indices[0]+1:indices[-1]] if len(indices) <= 1:
answer = results[0]
else:
answer = results[0][indices[0]+1:indices[-1]]
if self.is_equiv(answer, self.remove_boxed(self.last_boxed_only_string(doc["solution"]))): if self.is_equiv(answer, self.remove_boxed(self.last_boxed_only_string(doc["solution"]))):
retval = 1 retval = 1
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment