Merge pull request #520 from EleutherAI/update-config

Update config

lm_eval/tasks/super_glue/rte/promptsource-00.yaml

+group:
+  - super-glue-promptsource
+task: "GPT-3 style"
+dataset_path: super_glue
+dataset_name: rte
+training_split: train
+validation_split: validation
+use_prompt: "promptsource:GPT-3 style"
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true

lm_eval/tasks/super_glue/rte/promptsource-01.yaml

+include: promptsource-00.yaml
+group:
+  - super-glue-promptsource
+task: "MNLI crowdsource"
+use_prompt: "promptsource:MNLI crowdsource"

lm_eval/tasks/super_glue/rte/promptsource-02.yaml

+include: promptsource-00.yaml
+group:
+  - super-glue-promptsource
+task: "based on the previous passage"
+use_prompt: "promptsource:based on the previous passage"

lm_eval/tasks/super_glue/wic/promptsource-00.yaml

+group:
+  - super-glue-promptsource
+task: "GPT-3-prompt"
+dataset_path: super_glue
+dataset_name: wic
+training_split: train
+validation_split: validation
+use_prompt: "promptsource:GPT-3-prompt"
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true

lm_eval/tasks/super_glue/wic/promptsource-01.yaml

+include: promptsource-00.yaml
+group:
+  - super-glue-promptsource
+task: "GPT-3-prompt-with-label"
+use_prompt: "promptsource:GPT-3-prompt-with-label"

lm_eval/tasks/super_glue/wic/promptsource-02.yaml

+include: promptsource-00.yaml
+group:
+  - super-glue-promptsource
+task: "affirmation_true_or_false"
+use_prompt: "promptsource:affirmation_true_or_false"

lm_eval/tasks/super_glue/wsc.fixed/template-02.yaml → lm_eval/tasks/super_glue/wsc.fixed/promptsource-00.yaml

 group:
-  - t0-eval
-task: "in other words"
+  - super-glue-promptsource
+task: "GPT-3 Style"
 dataset_path: super_glue
 dataset_name: wsc.fixed
 training_split: train
 validation_split: validation
-use_prompt: "promptsource:in other words"
+use_prompt: "promptsource:GPT-3 Style"
 metric_list:
   - metric: exact_match
     aggregation: mean

lm_eval/tasks/super_glue/wsc.fixed/promptsource-01.yaml

+include: promptsource-00.yaml
+group:
+  - super-glue-promptsource
+task: "I think they mean"
+use_prompt: "promptsource:I think they mean"

lm_eval/tasks/super_glue/wsc.fixed/promptsource-02.yaml

+include: promptsource-00.yaml
+group:
+  - super-glue-promptsource
+task: "Who or what is/are"
+use_prompt: "promptsource:Who or what is/are"