moved templates, edit templates

lm_eval/benchmarks/flan/flan_held_in.yaml

-group: flan_held_in
-task:
-  - include: held_in_template_yaml
-    dataset_path: super_glue
-    dataset_name: boolq
-    use_prompt: prompt_templates/flan_boolq.yaml:*
-    validation_split: validation
-  - include: held_in_template_yaml
-    dataset_path: super_glue
-    dataset_name: rte
-    use_prompt: prompt_templates/flan_rte.yaml:*
-    validation_split: validation
-  - include: held_in_template_yaml
-    task: anli_r1
-    dataset_path: anli
-    use_prompt: prompt_templates/flan_anli.yaml:*
-    validation_split: dev_r1
-  - include: held_in_template_yaml
-    task: anli_r2
-    dataset_path: anli
-    use_prompt: prompt_templates/flan_anli.yaml:*
-    validation_split: dev_r2
-  - include: held_in_template_yaml
-    task: anli_r3
-    dataset_path: anli
-    use_prompt: prompt_templates/flan_anli.yaml:*
-    validation_split: dev_r3
-  # - include: held_in_template_yaml
-  #   task: ai2_arc
-  #   dataset_path: ARC-Easy
-  #   use_prompt: local:*
-  #   validation_split: validation
-  # - include: held_in_template_yaml
-  #   task: ai2_arc
-  #   dataset_path: ARC-Challange
-  #   use_prompt: local:*
-  #   validation_split: validation

lm_eval/benchmarks/flan/prompt_templates/flan_anli.yaml

 # Flan Prompt Templates
 prompts:
   "template-0":
-    doc_to_text: "{{context}}\n\nChoose your answer: based on the paragraph above can we conclude that \"{{hypothesis}}\"?\n\nOPTIONS:\n- Yes\n- It\'s impossible to say\n- No\nI think the answer is"
-    doc_to_target: "{{answer}}"
+    doc_to_text: "{{context}}\n\nChoose your answer: based on the paragraph above can we conclude that \"{{hypothesis}}\"?\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nI think the answer is"
+    doc_to_target: "{{[['Yes', 'It\'s impossible to say', 'No']][label]}}"
   "template-1":
-    doc_to_text: "{{context}}\n\nBased on that paragraph can we conclude that this sentence is true?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It\'s impossible to say\n- No"
-    doc_to_target: "{{answer}}"
+    doc_to_text: "{{context}}\n\nBased on that paragraph can we conclude that this sentence is true?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+    doc_to_target: "{{[['Yes', 'It\'s impossible to say', 'No']][label]}}"
   "template-2":
-    doc_to_text: "{{context}}\n\nCan we draw the following conclusion?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It\'s impossible to say\n- No"
-    doc_to_target: "{{answer}}"
+    doc_to_text: "{{context}}\n\nCan we draw the following conclusion?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+    doc_to_target: "{{[['Yes', 'It\'s impossible to say', 'No']][label]}}"
   "template-3":
-    doc_to_text: "{{context}}\nDoes this next sentence follow, given the preceding text?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It\'s impossible to say\n- No"
-    doc_to_target: "{{answer}}"
+    doc_to_text: "{{context}}\nDoes this next sentence follow, given the preceding text?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+    doc_to_target: "{{[['Yes', 'It\'s impossible to say', 'No']][label]}}"
   "template-4":
-    doc_to_text: "{{context}}\nCan we infer the following?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It\'s impossible to say\n- No\nThe answer is:"
-    doc_to_target: "{{answer}}"
+    doc_to_text: "{{context}}\nCan we infer the following?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nThe answer is:"
+    doc_to_target: "{{[['Yes', 'It\'s impossible to say', 'No']][label]}}"
   "template-5":
-    doc_to_text: "Read the following paragraph and determine if the hypothesis is true:\n\n{{context}}\n\nOPTIONS:\n- Yes\n- It\'s impossible to say\n- No\nHypothesis: {{hypothesis}}\n\n\n"
-    doc_to_target: "{{answer}}"
+    doc_to_text: "Read the following paragraph and determine if the hypothesis is true:\n\n{{context}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nHypothesis: {{hypothesis}}\n\n\n"
+    doc_to_target: "{{[['Yes', 'It\'s impossible to say', 'No']][label]}}"
   "template-6":
-    doc_to_text: "Read the text and determine if the sentence is true (see options at the end):\n\n{{context}}\n\nSentence: {{hypothesis}}\nOPTIONS:\n- Yes\n- It\'s impossible to say\n- No"
-    doc_to_target: "{{answer}}"
+    doc_to_text: "Read the text and determine if the sentence is true (see options at the end):\n\n{{context}}\n\nSentence: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+    doc_to_target: "{{[['Yes', 'It\'s impossible to say', 'No']][label]}}"
   "template-7":
-    doc_to_text: "Can we draw the following hypothesis from the context (see options)? \n\nContext:\n\n{{context}}\n\nHypothesis: {{hypothesis}}\nOPTIONS:\n- Yes\n- It\'s impossible to say\n- No"
-    doc_to_target: "{{answer}}"
+    doc_to_text: "Can we draw the following hypothesis from the context (see options)? \n\nContext:\n\n{{context}}\n\nHypothesis: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+    doc_to_target: "{{[['Yes', 'It\'s impossible to say', 'No']][label]}}"
   "template-8":
-    doc_to_text: "Choose from options: Determine if the sentence is true based on the text below:\n{{hypothesis}}\n\n{{context}}\nOPTIONS:\n- Yes\n- It\'s impossible to say\n- No"
-    doc_to_target: "{{answer}}"
+    doc_to_text: "Choose from options: Determine if the sentence is true based on the text below:\n{{hypothesis}}\n\n{{context}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+    doc_to_target: "{{[['Yes', 'It\'s impossible to say', 'No']][label]}}"

lm_eval/benchmarks/flan/prompt_templates/flan_boolq.yaml

 # Flan Prompt Templates
 prompts:
   "template-0":
-    doc_to_text: "{{text}}\n\nCan we conclude that {{question}}?\n\nOPTIONS:\n- no\n- yes"
-    doc_to_target: "{{answer}}"
+    doc_to_text: "{{passage}}\n\nCan we conclude that {{question}}?\n\nOPTIONS:\n- no\n- yes"
+    doc_to_target: "{{['no', 'yes'][label]}}"
   "template-1":
-    doc_to_text: "{{text}}\n\nIs it true that {{question}}?\n\nOPTIONS:\n- no\n- yes"
-    doc_to_target: "{{answer}}"
+    doc_to_text: "{{passage}}\n\nIs it true that {{question}}?\n\nOPTIONS:\n- no\n- yes"
+    doc_to_target: "{{['no', 'yes'][label]}}"
   "template-2":
-    doc_to_text: "{{text}}\n\n{{question}}?\n\nOPTIONS:\n- no\n- yes"
-    doc_to_target: "{{answer}}"
+    doc_to_text: "{{passage}}\n\n{{question}}?\n\nOPTIONS:\n- no\n- yes"
+    doc_to_target: "{{['no', 'yes'][label]}}"
   "template-3":
-    doc_to_text: "Text: {{text}}\n\nQuestion: {{question}}?\n\nOPTIONS:\n- no\n- yes"
-    doc_to_target: "{{answer}}"
+    doc_to_text: "Text: {{passage}}\n\nQuestion: {{question}}?\n\nOPTIONS:\n- no\n- yes"
+    doc_to_target: "{{['no', 'yes'][label]}}"
   "template-4":
-    doc_to_text: "{{text}}\n\nWhat's the best answer to this question: {{question}}?\n\nOPTIONS:\n- no\n- yes"
-    doc_to_target: "{{answer}}"
+    doc_to_text: "{{passage}}\n\nWhat's the best answer to this question: {{question}}?\n\nOPTIONS:\n- no\n- yes"
+    doc_to_target: "{{['no', 'yes'][label]}}"
   "template-5":
-    doc_to_text: "{{text}}\nBased on the above text what's the best answer to this question: {{question}}?\n\nOPTIONS:\n- no\n- yes"
-    doc_to_target: "{{answer}}"
+    doc_to_text: "{{passage}}\nBased on the above text what's the best answer to this question: {{question}}?\n\nOPTIONS:\n- no\n- yes"
+    doc_to_target: "{{['no', 'yes'][label]}}"
   "template-6":
-    doc_to_text: "{{text}}\nAnswer this question making sure that the answer is supposed by the text: {{question}}?\n\nOPTIONS:\n- no\n- yes"
-    doc_to_target: "{{answer}}"
+    doc_to_text: "{{passage}}\nAnswer this question making sure that the answer is supposed by the text: {{question}}?\n\nOPTIONS:\n- no\n- yes"
+    doc_to_target: "{{['no', 'yes'][label]}}"
   "template-7":
-    doc_to_text: "{{text}}\n\nIs the following statement correct based on the text\n\n{{question}}\n\nOPTIONS:\n- no\n- yes"
-    doc_to_target: "{{answer}}"
+    doc_to_text: "{{passage}}\n\nIs the following statement correct based on the text\n\n{{question}}\n\nOPTIONS:\n- no\n- yes"
+    doc_to_target: "{{['no', 'yes'][label]}}"
   "template-8":
-    doc_to_text: "{{title}}\n\n{{text}}\n\nIs this statement correct \"{{question}}\"?\n\nOPTIONS:\n- no\n- yes"
-    doc_to_target: "{{answer}}"
+    doc_to_text: "{{title}}\n\n{{passage}}\n\nIs this statement correct \"{{question}}\"?\n\nOPTIONS:\n- no\n- yes"
+    doc_to_target: "{{['no', 'yes'][label]}}"
   "template-9":
-    doc_to_text: "Is it true that {{question}} based on the following text?\n\n{{text}}\n\nOPTIONS:\n- no\n- yes"
-    doc_to_target: "{{answer}}"
+    doc_to_text: "Is it true that {{question}} based on the following text?\n\n{{passage}}\n\nOPTIONS:\n- no\n- yes"
+    doc_to_target: "{{['no', 'yes'][label]}}"

lm_eval/benchmarks/flan/prompt_templates/flan_rte.yaml

@@ -2,28 +2,28 @@
 prompts:
   "template-0":
     doc_to_text: "{{premise}}\n\nQuestion with options: Based on the paragraph above can we conclude that \"{{hypothesis}}\"?\n\nOPTIONS:\n- yes\n- no"
-    doc_to_target: "{{answer}}"
+    doc_to_target: "{{[['yes', 'no']][label]}}"
   "template-1":
     doc_to_text: "{{premise}}\n\nBased on that paragraph can we conclude that the sentence below is true?\n{{hypothesis}}\n\nOPTIONS:\n- yes\n- no"
-    doc_to_target: "{{answer}}"
+    doc_to_target: "{{[['yes', 'no']][label]}}"
   "template-2":
     doc_to_text: "{{premise}}\n\nQ with options: Can we draw the following conclusion?\n{{hypothesis}}\n\nOPTIONS:\n- yes\n- no"
-    doc_to_target: "{{answer}}"
+    doc_to_target: "{{[['yes', 'no']][label]}}"
   "template-3":
     doc_to_text: "{{premise}}\nDoes this next sentence follow, given the preceding text?\n{{hypothesis}}\n\nOPTIONS:\n- yes\n- no"
-    doc_to_target: "{{answer}}"
+    doc_to_target: "{{[['yes', 'no']][label]}}"
   "template-4":
     doc_to_text: "{{premise}}\nOPTIONS:\n- yes\n- no\nQuestion: Can we infer the following?\n{{hypothesis}}"
-    doc_to_target: "{{answer}}"
+    doc_to_target: "{{[['yes', 'no']][label]}}"
   "template-5":
     doc_to_text: "Read the following paragraph and determine if the hypothesis is true. Select from options at the end:\n\n{{premise}}\n\nHypothesis: {{hypothesis}}\nOPTIONS:\n- yes\n- no\nThe answer is"
-    doc_to_target: "{{answer}}"
+    doc_to_target: "{{[['yes', 'no']][label]}}"
   "template-6":
     doc_to_text: "Read the text and determine if the sentence is true:\n\n{{premise}}\n\nSentence: {{hypothesis}}\nOPTIONS:\n- yes\n- no\nA:"
-    doc_to_target: "{{answer}}"
+    doc_to_target: "{{[['yes', 'no']][label]}}"
   "template-7":
     doc_to_text: "Question with options: can we draw the following hypothesis from the context? \n\nContext:\n\n{{premise}}\n\nHypothesis: {{hypothesis}}\nOPTIONS:\n- yes\n- no\nA:"
-    doc_to_target: "{{answer}}"
+    doc_to_target: "{{[['yes', 'no']][label]}}"
   "template-8":
     doc_to_text: "Determine if the sentence is true based on the text below. Choose from options.\n{{hypothesis}}\n\n{{premise}}\nOPTIONS:\n- yes\n- no"
-    doc_to_target: "{{answer}}"
+    doc_to_target: "{{[['yes', 'no']][label]}}"

lm_eval/benchmarks/flan/cot_template_yaml → lm_eval/benchmarks/flan/yaml_templates/cot_template_yaml

-group: zero-shot-cot
+group: flan-cot
 output_type: greedy_until
 validation_split: validation
 doc_to_target: "{{answer}}"