added more promptsource examples

lm_eval/tasks/super_glue/wsc.fixed/template-00.yaml → lm_eval/tasks/super_glue/boolq/promptsource-00.yaml

 group:
-  - t0-eval
-task: "does the pronoun refer to"
+  - super-glue-promptsource
+task: "GPT-3 Style"
 dataset_path: super_glue
-dataset_name: wsc.fixed
+dataset_name: boolq
 training_split: train
 validation_split: validation
-use_prompt: "promptsource:does the pronoun refer to"
+use_prompt: "promptsource:GPT-3 Style"
 metric_list:
   - metric: exact_match
     aggregation: mean

lm_eval/tasks/super_glue/boolq/promptsource-01.yaml

+include: promptsource-00.yaml
+group:
+  - super-glue-promptsource
+task: "based on the previous passage"
+use_prompt: "promptsource:based on the previous passage"

lm_eval/tasks/super_glue/boolq/promptsource-02.yaml

+include: promptsource-00.yaml
+group:
+  - super-glue-promptsource
+task: "based on the following passage"
+use_prompt: "promptsource:based on the following passage"

lm_eval/tasks/super_glue/cb/can_we_infer.yaml

-group:
-  - super-glue-cb
-include: based_on_previous_passage.yaml
-task: can we infer
-reference: Webson & Pavlick 2021
-doc_to_text: "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe?"
-doc_to_target: "{% set answer_choices = ['Yes', 'No', 'Maybe'] %}{{answer_choices[label]}}"

lm_eval/tasks/super_glue/cb/claim_true_false_inconclusive.yaml

-group:
-  - super-glue-cb
-include: based_on_previous_passage.yaml
-task: claim true/false/inconclusive
-reference: Sanh et al. 2021
-doc_to_text: "{{premise}} Based on that information, is the claim: \"{{hypothesis}}\" \"true\", \"false\", or \"inconclusive\"?"
-doc_to_target: "{% set answer_choices = ['True', 'False', 'Inconclusive'] %}{{answer_choices[label]}}"

lm_eval/tasks/super_glue/wsc.fixed/template-01.yaml → lm_eval/tasks/super_glue/cb/promptsource-00.yaml

 group:
-  - t0-eval
-task: "by p they mean"
+  - super-glue-promptsource
+task: "GPT-3 style"
 dataset_path: super_glue
-dataset_name: wsc.fixed
+dataset_name: cb
 training_split: train
 validation_split: validation
-use_prompt: "promptsource:by p they mean"
+use_prompt: "promptsource:GPT-3 style"
 metric_list:
   - metric: exact_match
     aggregation: mean

lm_eval/tasks/super_glue/cb/promptsource-01.yaml

+include: promptsource-00.yaml
+group:
+  - super-glue-promptsource
+task: "MNLI crowdsource"
+use_prompt: "promptsource:MNLI crowdsource"

lm_eval/tasks/super_glue/cb/promptsource-02.yaml

+include: promptsource-00.yaml
+group:
+  - super-glue-promptsource
+task: "based on the previous passage"
+use_prompt: "promptsource:based on the previous passage"

lm_eval/tasks/super_glue/cb/based_on_previous_passage.yaml → lm_eval/tasks/super_glue/copa/promptsource-00.yaml

 group:
-  - super-glue-cb
-task: based on the previous passage
-reference: "Adapted from the BoolQ prompts in Schick & Sch\xFCtze 2021."
+  - super-glue-promptsource
+task: "C1 or C2? premise, so/because…"
 dataset_path: super_glue
-dataset_name: cb
+dataset_name: copa
 training_split: train
 validation_split: validation
-doc_to_text: "{{premise}} Based on the previous passage, is it true that \"{{hypothesis}}\"? Yes, no, or maybe?"
-doc_to_target: "{% set answer_choices = ['Yes', 'No', 'Maybe'] %}{{answer_choices[label]}}"
+use_prompt: "promptsource:C1 or C2? premise, so/because…"
 metric_list:
   - metric: exact_match
     aggregation: mean

lm_eval/tasks/super_glue/copa/promptsource-01.yaml

+include: promptsource-00.yaml
+group:
+  - super-glue-promptsource
+task: "best_option"
+use_prompt: "promptsource:best_option"

lm_eval/tasks/super_glue/copa/promptsource-02.yaml

+include: promptsource-00.yaml
+group:
+  - super-glue-promptsource
+task: "cause_effect"
+use_prompt: "promptsource:cause_effect"

lm_eval/tasks/super_glue/multirc/promptsource-00.yaml

+group:
+  - super-glue-promptsource
+task: "I was going to say…"
+dataset_path: super_glue
+dataset_name: multirc
+training_split: train
+validation_split: validation
+use_prompt: "promptsource:I was going to say…"
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true

lm_eval/tasks/super_glue/multirc/promptsource-01.yaml

+include: promptsource-00.yaml
+group:
+  - super-glue-promptsource
+task: "Would it be good to answer…"
+use_prompt: "promptsource:Would it be good to answer…"

lm_eval/tasks/super_glue/multirc/promptsource-02.yaml

+include: promptsource-00.yaml
+group:
+  - super-glue-promptsource
+task: "confirm"
+use_prompt: "promptsource:confirm"

lm_eval/tasks/super_glue/record/promptsource-00.yaml

+group:
+  - super-glue-promptsource
+task: "Add sentence after (continuation choices)"
+dataset_path: super_glue
+dataset_name: record
+training_split: train
+validation_split: validation
+use_prompt: "promptsource:Add sentence after (continuation choices)"
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true

lm_eval/tasks/super_glue/record/promptsource-01.yaml

+include: promptsource-00.yaml
+group:
+  - super-glue-promptsource
+task: "Add sentence after after (continuation choices)"
+use_prompt: "promptsource:Add sentence after after (continuation choices)"

lm_eval/tasks/super_glue/record/promptsource-02.yaml

+include: promptsource-00.yaml
+group:
+  - super-glue-promptsource
+task: "Can you figure out…"
+use_prompt: "promptsource:Can you figure out…"

lm_eval/tasks/super_glue/rte/promptsource-00.yaml

+group:
+  - super-glue-promptsource
+task: "GPT-3 style"
+dataset_path: super_glue
+dataset_name: rte
+training_split: train
+validation_split: validation
+use_prompt: "promptsource:GPT-3 style"
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true

lm_eval/tasks/super_glue/rte/promptsource-01.yaml

+include: promptsource-00.yaml
+group:
+  - super-glue-promptsource
+task: "MNLI crowdsource"
+use_prompt: "promptsource:MNLI crowdsource"