added subtasks

lm_eval/benchmarks/flan/flan_held_in.yaml

@@ -10,21 +10,21 @@ task:
     dataset_name: rte
     use_prompt: prompt_templates/flan_rte.yaml:*
     validation_split: validation
-  # - include: held_in_template_yaml
-  #   task: anli_r1
-  #   dataset_path: anli
-  #   use_prompt: local:*
-  #   validation_split: dev_r1
-  # - include: held_in_template_yaml
-  #   task: anli_r2
-  #   dataset_path: anli
-  #   use_prompt: local:*
-  #   validation_split: dev_r2
-  # - include: held_in_template_yaml
-  #   task: anli_r3
-  #   dataset_path: anli
-  #   use_prompt: local:*
-  #   validation_split: dev_r3
+  - include: held_in_template_yaml
+    task: anli_r1
+    dataset_path: anli
+    use_prompt: prompt_templates/flan_anli.yaml:*
+    validation_split: dev_r1
+  - include: held_in_template_yaml
+    task: anli_r2
+    dataset_path: anli
+    use_prompt: prompt_templates/flan_anli.yaml:*
+    validation_split: dev_r2
+  - include: held_in_template_yaml
+    task: anli_r3
+    dataset_path: anli
+    use_prompt: prompt_templates/flan_anli.yaml:*
+    validation_split: dev_r3
   # - include: held_in_template_yaml
   #   task: ai2_arc
   #   dataset_path: ARC-Easy

lm_eval/benchmarks/flan/prompt_templates/flan_anli.yaml

+# Flan Prompt Templates
+prompts:
+  "template-0":
+    doc_to_text: "{{context}}\n\nChoose your answer: based on the paragraph above can we conclude that \"{{hypothesis}}\"?\n\nOPTIONS:\n- Yes\n- It\'s impossible to say\n- No\nI think the answer is"
+    doc_to_target: "{{answer}}"
+  "template-1":
+    doc_to_text: "{{context}}\n\nBased on that paragraph can we conclude that this sentence is true?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It\'s impossible to say\n- No"
+    doc_to_target: "{{answer}}"
+  "template-2":
+    doc_to_text: "{{context}}\n\nCan we draw the following conclusion?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It\'s impossible to say\n- No"
+    doc_to_target: "{{answer}}"
+  "template-3":
+    doc_to_text: "{{context}}\nDoes this next sentence follow, given the preceding text?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It\'s impossible to say\n- No"
+    doc_to_target: "{{answer}}"
+  "template-4":
+    doc_to_text: "{{context}}\nCan we infer the following?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It\'s impossible to say\n- No\nThe answer is:"
+    doc_to_target: "{{answer}}"
+  "template-5":
+    doc_to_text: "Read the following paragraph and determine if the hypothesis is true:\n\n{{context}}\n\nOPTIONS:\n- Yes\n- It\'s impossible to say\n- No\nHypothesis: {{hypothesis}}\n\n\n"
+    doc_to_target: "{{answer}}"
+  "template-6":
+    doc_to_text: "Read the text and determine if the sentence is true (see options at the end):\n\n{{context}}\n\nSentence: {{hypothesis}}\nOPTIONS:\n- Yes\n- It\'s impossible to say\n- No"
+    doc_to_target: "{{answer}}"
+  "template-7":
+    doc_to_text: "Can we draw the following hypothesis from the context (see options)? \n\nContext:\n\n{{context}}\n\nHypothesis: {{hypothesis}}\nOPTIONS:\n- Yes\n- It\'s impossible to say\n- No"
+    doc_to_target: "{{answer}}"
+  "template-8":
+    doc_to_text: "Choose from options: Determine if the sentence is true based on the text below:\n{{hypothesis}}\n\n{{context}}\nOPTIONS:\n- Yes\n- It\'s impossible to say\n- No"
+    doc_to_target: "{{answer}}"

lm_eval/benchmarks/flan/prompt_templates/flan_rte.yaml

+# Flan Prompt Templates
+prompts:
+  "template-0":
+    doc_to_text: "{{premise}}\n\nQuestion with options: Based on the paragraph above can we conclude that \"{{hypothesis}}\"?\n\nOPTIONS:\n- yes\n- no"
+    doc_to_target: "{{answer}}"
+  "template-1":
+    doc_to_text: "{{premise}}\n\nBased on that paragraph can we conclude that the sentence below is true?\n{{hypothesis}}\n\nOPTIONS:\n- yes\n- no"
+    doc_to_target: "{{answer}}"
+  "template-2":
+    doc_to_text: "{{premise}}\n\nQ with options: Can we draw the following conclusion?\n{{hypothesis}}\n\nOPTIONS:\n- yes\n- no"
+    doc_to_target: "{{answer}}"
+  "template-3":
+    doc_to_text: "{{premise}}\nDoes this next sentence follow, given the preceding text?\n{{hypothesis}}\n\nOPTIONS:\n- yes\n- no"
+    doc_to_target: "{{answer}}"
+  "template-4":
+    doc_to_text: "{{premise}}\nOPTIONS:\n- yes\n- no\nQuestion: Can we infer the following?\n{{hypothesis}}"
+    doc_to_target: "{{answer}}"
+  "template-5":
+    doc_to_text: "Read the following paragraph and determine if the hypothesis is true. Select from options at the end:\n\n{{premise}}\n\nHypothesis: {{hypothesis}}\nOPTIONS:\n- yes\n- no\nThe answer is"
+    doc_to_target: "{{answer}}"
+  "template-6":
+    doc_to_text: "Read the text and determine if the sentence is true:\n\n{{premise}}\n\nSentence: {{hypothesis}}\nOPTIONS:\n- yes\n- no\nA:"
+    doc_to_target: "{{answer}}"
+  "template-7":
+    doc_to_text: "Question with options: can we draw the following hypothesis from the context? \n\nContext:\n\n{{premise}}\n\nHypothesis: {{hypothesis}}\nOPTIONS:\n- yes\n- no\nA:"
+    doc_to_target: "{{answer}}"
+  "template-8":
+    doc_to_text: "Determine if the sentence is true based on the text below. Choose from options.\n{{hypothesis}}\n\n{{premise}}\nOPTIONS:\n- yes\n- no"
+    doc_to_target: "{{answer}}"