updated yaml templates

lm_eval/benchmarks/flan/prompt_templates/flan_anli.yaml

@@ -2,28 +2,28 @@
 prompts:
   "template-0":
     doc_to_text: "{{context}}\n\nChoose your answer: based on the paragraph above can we conclude that \"{{hypothesis}}\"?\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nI think the answer is"
-    doc_to_target: "{{[['Yes', 'It\'s impossible to say', 'No']][label]}}"
+    doc_to_target: "{{['Yes', 'It\'s impossible to say', 'No'][label]}}"
   "template-1":
     doc_to_text: "{{context}}\n\nBased on that paragraph can we conclude that this sentence is true?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
-    doc_to_target: "{{[['Yes', 'It\'s impossible to say', 'No']][label]}}"
+    doc_to_target: "{{['Yes', 'It\'s impossible to say', 'No'][label]}}"
   "template-2":
     doc_to_text: "{{context}}\n\nCan we draw the following conclusion?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
-    doc_to_target: "{{[['Yes', 'It\'s impossible to say', 'No']][label]}}"
+    doc_to_target: "{{['Yes', 'It\'s impossible to say', 'No'][label]}}"
   "template-3":
     doc_to_text: "{{context}}\nDoes this next sentence follow, given the preceding text?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
-    doc_to_target: "{{[['Yes', 'It\'s impossible to say', 'No']][label]}}"
+    doc_to_target: "{{['Yes', 'It\'s impossible to say', 'No'][label]}}"
   "template-4":
     doc_to_text: "{{context}}\nCan we infer the following?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nThe answer is:"
-    doc_to_target: "{{[['Yes', 'It\'s impossible to say', 'No']][label]}}"
+    doc_to_target: "{{['Yes', 'It\'s impossible to say', 'No'][label]}}"
   "template-5":
     doc_to_text: "Read the following paragraph and determine if the hypothesis is true:\n\n{{context}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nHypothesis: {{hypothesis}}\n\n\n"
-    doc_to_target: "{{[['Yes', 'It\'s impossible to say', 'No']][label]}}"
+    doc_to_target: "{{['Yes', 'It\'s impossible to say', 'No'][label]}}"
   "template-6":
     doc_to_text: "Read the text and determine if the sentence is true (see options at the end):\n\n{{context}}\n\nSentence: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
-    doc_to_target: "{{[['Yes', 'It\'s impossible to say', 'No']][label]}}"
+    doc_to_target: "{{['Yes', 'It\'s impossible to say', 'No'][label]}}"
   "template-7":
     doc_to_text: "Can we draw the following hypothesis from the context (see options)? \n\nContext:\n\n{{context}}\n\nHypothesis: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
-    doc_to_target: "{{[['Yes', 'It\'s impossible to say', 'No']][label]}}"
+    doc_to_target: "{{['Yes', 'It\'s impossible to say', 'No'][label]}}"
   "template-8":
     doc_to_text: "Choose from options: Determine if the sentence is true based on the text below:\n{{hypothesis}}\n\n{{context}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
-    doc_to_target: "{{[['Yes', 'It\'s impossible to say', 'No']][label]}}"
+    doc_to_target: "{{['Yes', 'It\'s impossible to say', 'No'][label]}}"

lm_eval/benchmarks/flan/prompt_templates/flan_boolq.yaml

@@ -25,7 +25,8 @@ prompts:
     doc_to_text: "{{passage}}\n\nIs the following statement correct based on the text\n\n{{question}}\n\nOPTIONS:\n- no\n- yes"
     doc_to_target: "{{['no', 'yes'][label]}}"
   "template-8":
-    doc_to_text: "{{title}}\n\n{{passage}}\n\nIs this statement correct \"{{question}}\"?\n\nOPTIONS:\n- no\n- yes"
+    # doc_to_text: "{{title}}\n\n{{passage}}\n\nIs this statement correct \"{{question}}\"?\n\nOPTIONS:\n- no\n- yes"
+    doc_to_text: "{{passage}}\n\nIs this statement correct \"{{question}}\"?\n\nOPTIONS:\n- no\n- yes"
     doc_to_target: "{{['no', 'yes'][label]}}"
   "template-9":
     doc_to_text: "Is it true that {{question}} based on the following text?\n\n{{passage}}\n\nOPTIONS:\n- no\n- yes"

lm_eval/benchmarks/flan/prompt_templates/flan_rte.yaml

@@ -2,28 +2,28 @@
 prompts:
   "template-0":
     doc_to_text: "{{premise}}\n\nQuestion with options: Based on the paragraph above can we conclude that \"{{hypothesis}}\"?\n\nOPTIONS:\n- yes\n- no"
-    doc_to_target: "{{[['yes', 'no']][label]}}"
+    doc_to_target: "{{['yes', 'no'][label]}}"
   "template-1":
     doc_to_text: "{{premise}}\n\nBased on that paragraph can we conclude that the sentence below is true?\n{{hypothesis}}\n\nOPTIONS:\n- yes\n- no"
-    doc_to_target: "{{[['yes', 'no']][label]}}"
+    doc_to_target: "{{['yes', 'no'][label]}}"
   "template-2":
     doc_to_text: "{{premise}}\n\nQ with options: Can we draw the following conclusion?\n{{hypothesis}}\n\nOPTIONS:\n- yes\n- no"
-    doc_to_target: "{{[['yes', 'no']][label]}}"
+    doc_to_target: "{{['yes', 'no'][label]}}"
   "template-3":
     doc_to_text: "{{premise}}\nDoes this next sentence follow, given the preceding text?\n{{hypothesis}}\n\nOPTIONS:\n- yes\n- no"
-    doc_to_target: "{{[['yes', 'no']][label]}}"
+    doc_to_target: "{{['yes', 'no'][label]}}"
   "template-4":
     doc_to_text: "{{premise}}\nOPTIONS:\n- yes\n- no\nQuestion: Can we infer the following?\n{{hypothesis}}"
-    doc_to_target: "{{[['yes', 'no']][label]}}"
+    doc_to_target: "{{['yes', 'no'][label]}}"
   "template-5":
     doc_to_text: "Read the following paragraph and determine if the hypothesis is true. Select from options at the end:\n\n{{premise}}\n\nHypothesis: {{hypothesis}}\nOPTIONS:\n- yes\n- no\nThe answer is"
-    doc_to_target: "{{[['yes', 'no']][label]}}"
+    doc_to_target: "{{['yes', 'no'][label]}}"
   "template-6":
     doc_to_text: "Read the text and determine if the sentence is true:\n\n{{premise}}\n\nSentence: {{hypothesis}}\nOPTIONS:\n- yes\n- no\nA:"
-    doc_to_target: "{{[['yes', 'no']][label]}}"
+    doc_to_target: "{{['yes', 'no'][label]}}"
   "template-7":
     doc_to_text: "Question with options: can we draw the following hypothesis from the context? \n\nContext:\n\n{{premise}}\n\nHypothesis: {{hypothesis}}\nOPTIONS:\n- yes\n- no\nA:"
-    doc_to_target: "{{[['yes', 'no']][label]}}"
+    doc_to_target: "{{['yes', 'no'][label]}}"
   "template-8":
     doc_to_text: "Determine if the sentence is true based on the text below. Choose from options.\n{{hypothesis}}\n\n{{premise}}\nOPTIONS:\n- yes\n- no"
-    doc_to_target: "{{[['yes', 'no']][label]}}"
+    doc_to_target: "{{['yes', 'no'][label]}}"