added more flan subtasks

lm_eval/benchmarks/flan/flan_cot_yaml

-group: flan_cot
-task:
-  - include: cot_template_yaml
-    dataset_path: super_glue
-    dataset_name: boolq
-    use_prompt: promptsource:*
-    validation_split: validation
-  - include: cot_template_yaml
-    dataset_path: super_glue
-    dataset_name: rte
-    use_prompt: promptsource:*
-    validation_split: validation
-  - include: cot_template_yaml
-    task: anli_r1
-    dataset_path: anli
-    use_prompt: promptsource:*
-    validation_split: dev_r1
-  - include: cot_template_yaml
-    task: anli_r2
-    dataset_path: anli
-    use_prompt: promptsource:*
-    validation_split: dev_r2
-  - include: cot_template_yaml
-    task: anli_r3
-    dataset_path: anli
-    use_prompt: promptsource:*
-    validation_split: dev_r3
-  - include: cot_template_yaml
-    task: ai2_arc
-    dataset_path: ARC-Easy
-    use_prompt: promptsource:*
-    validation_split: validation
-  - include: cot_template_yaml
-    task: ai2_arc
-    dataset_path: ARC-Challange
-    use_prompt: promptsource:*
-    validation_split: validation

lm_eval/benchmarks/flan/prompt_templates/flan_bbh.yaml

+# Flan Prompt Templates
+prompts:
+  "template-0":
+    doc_to_text: "{{context}}\n\nChoose your answer: based on the paragraph above can we conclude that \"{{hypothesis}}\"?\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nI think the answer is"
+    doc_to_target: "{{['Yes', 'It\'s impossible to say', 'No'][label]}}"
+  "template-1":
+    doc_to_text: "{{context}}\n\nBased on that paragraph can we conclude that this sentence is true?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+    doc_to_target: "{{['Yes', 'It\'s impossible to say', 'No'][label]}}"
+  "template-2":
+    doc_to_text: "{{context}}\n\nCan we draw the following conclusion?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+    doc_to_target: "{{['Yes', 'It\'s impossible to say', 'No'][label]}}"
+  "template-3":
+    doc_to_text: "{{context}}\nDoes this next sentence follow, given the preceding text?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+    doc_to_target: "{{['Yes', 'It\'s impossible to say', 'No'][label]}}"
+  "template-4":
+    doc_to_text: "{{context}}\nCan we infer the following?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nThe answer is:"
+    doc_to_target: "{{['Yes', 'It\'s impossible to say', 'No'][label]}}"
+  "template-5":
+    doc_to_text: "Read the following paragraph and determine if the hypothesis is true:\n\n{{context}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nHypothesis: {{hypothesis}}\n\n\n"
+    doc_to_target: "{{['Yes', 'It\'s impossible to say', 'No'][label]}}"
+  "template-6":
+    doc_to_text: "Read the text and determine if the sentence is true (see options at the end):\n\n{{context}}\n\nSentence: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+    doc_to_target: "{{['Yes', 'It\'s impossible to say', 'No'][label]}}"
+  "template-7":
+    doc_to_text: "Can we draw the following hypothesis from the context (see options)? \n\nContext:\n\n{{context}}\n\nHypothesis: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+    doc_to_target: "{{['Yes', 'It\'s impossible to say', 'No'][label]}}"
+  "template-8":
+    doc_to_text: "Choose from options: Determine if the sentence is true based on the text below:\n{{hypothesis}}\n\n{{context}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+    doc_to_target: "{{['Yes', 'It\'s impossible to say', 'No'][label]}}"

lm_eval/benchmarks/flan_cot.yaml

+group: flan_cot
+task:
+  - include: flan/yaml_templates/cot_template_yaml
+    dataset_path: gsmk
+    dataset_name: boolq
+    use_prompt: promptsource:*
+    validation_split: validation
+  - include: flan/yaml_templates/cot_template_yaml
+    dataset_path: EleutherAI/asdiv
+    use_prompt: promptsource:*
+    validation_split: validation

lm_eval/benchmarks/flan_held_in.yaml

@@ -5,32 +5,32 @@ task:
     dataset_name: boolq
     use_prompt: flan/prompt_templates/flan_boolq.yaml:*
     validation_split: validation
-  - include: flan/yaml_templates//held_in_template_yaml
+  - include: flan/yaml_templates/held_in_template_yaml
     dataset_path: super_glue
     dataset_name: rte
     use_prompt: flan/prompt_templates/flan_rte.yaml:*
     validation_split: validation
-  - include: flan/yaml_templates//held_in_template_yaml
+  - include: flan/yaml_templates/held_in_template_yaml
     task: anli_r1
     dataset_path: anli
     use_prompt: flan/prompt_templates/flan_anli.yaml:*
     validation_split: dev_r1
-  - include: flan/yaml_templates//held_in_template_yaml
+  - include: flan/yaml_templates/held_in_template_yaml
     task: anli_r2
     dataset_path: anli
     use_prompt: flan/prompt_templates/flan_anli.yaml:*
     validation_split: dev_r2
-  - include: flan/yaml_templates//held_in_template_yaml
+  - include: flan/yaml_templates/held_in_template_yaml
     task: anli_r3
     dataset_path: anli
     use_prompt: flan/prompt_templates/flan_anli.yaml:*
     validation_split: dev_r3
-  # - include: flan/yaml_templates//held_in_template_yaml
+  # - include: flan/yaml_templates/held_in_template_yaml
   #   task: ai2_arc
   #   dataset_path: ARC-Easy
   #   use_prompt: local:*
   #   validation_split: validation
-  # - include: flan/yaml_templates//held_in_template_yaml
+  # - include: flan/yaml_templates/held_in_template_yaml
   #   task: ai2_arc
   #   dataset_path: ARC-Challange
   #   use_prompt: local:*

lm_eval/benchmarks/flan_held_out.yaml

+group: flan_held_out
+task:
+  - bbh
+  - mmlu