updated yaml

lm_eval/tasks/arc.yaml

@@ -5,6 +5,7 @@ validation_split: validation
 test_split: test
 doc_to_text: "Q: {{question}}\nA:"
 doc_to_target: "{% set answer_choices = doc['choices']['text'] %}{{answer_choices[int(doc['answerKey']) - 1]}}"
-metric_list: [
-  [exact_match, mean, true]
-  ]
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true

lm_eval/tasks/gsm8k.yaml

@@ -2,12 +2,15 @@ dataset_path: gsm8k
 dataset_name: main
 training_split: train
 test_split: test
-doc_to_text: "Question: {{question}}\nAnswer:"
-doc_to_target: "{{answer}}" # TODO: this field needs to change to account for the regexing that happens etc.
-metric_list: [
-  [acc, mean, true]
-  ]
-filters: [
-  ["regex", ["regex", "take_first"]]
-]
-stop_sequences: ["\n"]
+doc_to_target: "{{answer.split('### ')[-1]}}"
+use_prompt: "qa-basic:question-newline-answer"
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+delimiter: "\n"
\ No newline at end of file
+# filters: [
+#   ["regex", ["regex", "take_first"]]
+# ]
\ No newline at end of file