fix typos

lm_eval/filters/__init__.py

@@ -9,6 +9,7 @@ FILTER_REGISTRY = {
     "majority_vote": selection.MajorityVoteFilter,
     "take_first_k": selection.TakeKFilter,
     "remove_whitespace": extraction.WhitespaceFilter,
+    "cot_filter": extraction.CoTFilter,
     # TODO: implement this filter. either it should take in an arbitrary "scoring"/reward function
     # that takes an input and returns a scalar and then should select the max reward,
     # or should implement different filters for different ways of handling a reward model's inference.

lm_eval/filters/extraction.py

@@ -59,3 +59,45 @@ class WhitespaceFilter(Filter):
         filtered_resps = [filter_set(resp) for resp in resps]
 
         return filtered_resps
+
+
+class CoTFilter(Filter):
+    """ """
+
+    def __init__(self):
+        pass
+
+    def apply(self, resps):
+        def filter_set(inst):
+
+            filtered_resp = []
+            for resp in inst:
+
+                resp = resp.strip()
+                if resp[-1] in [".", ",", "?", " ", "\n"]:
+                    resp = resp[:-1].strip()
+
+                if resp[0] == "(" and resp[-1] == ")":
+                    resp = resp[1:-1].strip()
+                    return resp
+                else:
+                    resp = resp.split("resp is")[-1].strip()
+                    resp = resp.split("final resp")[-1].strip()
+                    resp = resp.split("Final resp")[-1].strip()
+                    resp = resp.split("resp:")[-1].strip()
+                    resp = resp.split("resp:")[-1].strip()
+                    if resp and resp[0] in [".", ",", "?", " ", "\n", ":"]:
+                        resp = resp[1:].strip()
+                    if resp and resp[-1] in [".", ",", "?", " ", "\n", ":"]:
+                        resp = resp[:-1].strip()
+                    # corner case 2: is prediction is (B), should processed into B.
+                    if resp and resp[0] == "(" and resp[-1] == ")":
+                        resp = resp[1:-1].strip()
+
+                filtered_resp.append(resp)
+
+            return filtered_resp
+
+        filtered_resps = [filter_set(resp) for resp in resps]
+
+        return filtered_resps

lm_eval/tasks/model_written_evals/persona/template_yaml

 group: persona
 dataset_path: lm_eval.datasets.model_written_evals.persona
 output_type: multiple_choice
-validation_split: train
+validation_split: validation
+target_delimiter: ""
 doc_to_text: "{{question}}"
 doc_to_target: 0
 doc_to_choice: "{{[answer_matching_behavior, answer_not_matching_behavior]}}"