udpated to work

lm_eval/benchmarks/flan/flan_held_in.yaml

@@ -3,13 +3,13 @@ task:
   - include: held_in_template_yaml
     dataset_path: super_glue
     dataset_name: boolq
-    use_prompt: flan_boolq.yaml:*
+    use_prompt: prompt_templates/flan_boolq.yaml:*
+    validation_split: validation
+  - include: held_in_template_yaml
+    dataset_path: super_glue
+    dataset_name: rte
+    use_prompt: prompt_templates/flan_rte.yaml:*
     validation_split: validation
-  # - include: held_in_template_yaml
-  #   dataset_path: super_glue
-  #   dataset_name: rte
-  #   use_prompt: local:*
-  #   validation_split: validation
   # - include: held_in_template_yaml
   #   task: anli_r1
   #   dataset_path: anli

lm_eval/benchmarks/flan/flan_boolq.yaml → lm_eval/benchmarks/flan/prompt_templates/flan_boolq.yaml

 # Flan Prompt Templates
 prompts:
   "template-0":
-    doc_to_text: "{{text}}\n\nCan we conclude that {{question}}?\n\n{{options_}}"
+    doc_to_text: "{{text}}\n\nCan we conclude that {{question}}?\n\nOPTIONS:\n- no\n- yes"
     doc_to_target: "{{answer}}"
   "template-1":
-    doc_to_text: "{{text}}\n\nIs it true that {{question}}?\n\n{{options_}}"
+    doc_to_text: "{{text}}\n\nIs it true that {{question}}?\n\nOPTIONS:\n- no\n- yes"
     doc_to_target: "{{answer}}"
   "template-2":
-    doc_to_text: "{{text}}\n\n{{question}}?\n\n{{options_}}"
+    doc_to_text: "{{text}}\n\n{{question}}?\n\nOPTIONS:\n- no\n- yes"
     doc_to_target: "{{answer}}"
   "template-3":
-    doc_to_text: "Text: {{text}}\n\nQuestion: {{question}}?\n\n{{options_}}"
+    doc_to_text: "Text: {{text}}\n\nQuestion: {{question}}?\n\nOPTIONS:\n- no\n- yes"
     doc_to_target: "{{answer}}"
   "template-4":
-    doc_to_text: "{{text}}\n\nWhat's the best answer to this question: {{question}}?\n\n{{options_}}"
+    doc_to_text: "{{text}}\n\nWhat's the best answer to this question: {{question}}?\n\nOPTIONS:\n- no\n- yes"
     doc_to_target: "{{answer}}"
   "template-5":
-    doc_to_text: "{{text}}\nBased on the above text what's the best answer to this question: {{question}}?\n\n{{options_}}"
+    doc_to_text: "{{text}}\nBased on the above text what's the best answer to this question: {{question}}?\n\nOPTIONS:\n- no\n- yes"
     doc_to_target: "{{answer}}"
   "template-6":
-    doc_to_text: "{{text}}\nAnswer this question making sure that the answer is supposed by the text: {{question}}?\n\n{{options_}}"
+    doc_to_text: "{{text}}\nAnswer this question making sure that the answer is supposed by the text: {{question}}?\n\nOPTIONS:\n- no\n- yes"
     doc_to_target: "{{answer}}"
   "template-7":
-    doc_to_text: "{{text}}\n\nIs the following statement correct based on the text\n\n{{question}}\n\n{{options_}}"
+    doc_to_text: "{{text}}\n\nIs the following statement correct based on the text\n\n{{question}}\n\nOPTIONS:\n- no\n- yes"
     doc_to_target: "{{answer}}"
   "template-8":
-    doc_to_text: "{{title}}\n\n{{text}}\n\nIs this statement correct \"{{question}}\"?\n\n{{options_}}"
+    doc_to_text: "{{title}}\n\n{{text}}\n\nIs this statement correct \"{{question}}\"?\n\nOPTIONS:\n- no\n- yes"
     doc_to_target: "{{answer}}"
   "template-9":
-    doc_to_text: "Is it true that {{question}} based on the following text?\n\n{{text}}\n\n{{options_}}"
+    doc_to_text: "Is it true that {{question}} based on the following text?\n\n{{text}}\n\nOPTIONS:\n- no\n- yes"
     doc_to_target: "{{answer}}"

lm_eval/prompts/__init__.py

@@ -92,7 +92,7 @@ def load_prompt_list(use_prompt: str, dataset_name=None, subset_name=None, **kwa
 
 
 class PromptString:
-    def __init__(prompt_string):
+    def __init__(self, prompt_string):
         self.prompt_string = prompt_string
 
     def apply(self, doc):