Merge branch 'main' into multimodal-prototyping

lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_political_typology_quiz.yaml

-group: sycophancy
+tag: sycophancy
 task: sycophancy_on_political_typology_quiz
 dataset_path: EleutherAI/sycophancy
 dataset_name: sycophancy_on_political_typology_quiz

lm_eval/tasks/model_written_evals/winogenerated/_template_yaml → lm_eval/tasks/model_written_evals/winogenerated/winogenerated.yaml

-group: winogenerated
+tag: winogenerated
 dataset_path: EleutherAI/winogenerated
 output_type: multiple_choice
 validation_split: validation

lm_eval/tasks/okapi/arc_multilingual/_arc_yaml

-group:
+tag:
   - arc_multilingual
 dataset_path: null
 dataset_name: null

lm_eval/tasks/okapi/hellaswag_multilingual/_hellaswag_yaml

-group:
+tag:
   - hellaswag_multilingual
 dataset_path: null
 dataset_name: null

lm_eval/tasks/okapi/mmlu_multilingual/_default_yaml

-group:
+tag:
   - m_mmlu
 dataset_path: alexandrainst/m_mmlu
 test_split: test

lm_eval/tasks/okapi/truthfulqa_multilingual/_truthfulqa_mc1_yaml

-group:
+tag:
   - truthfulqa_multilingual
 dataset_path: null
 dataset_name: null

lm_eval/tasks/paloma/_paloma_template

-group:
+tag:
   - paloma
 dataset_path: allenai/paloma
 output_type: loglikelihood_rolling

lm_eval/tasks/paws-x/_pawsx.yaml

+group: pawsx
+task:
+  - paws_en
+  - paws_de
+  - paws_es
+  - paws_fr
+  - paws_ja
+  - paws_ko
+  - paws_zh
+aggregate_metric_list:
+  - metric: acc
+    aggregation: mean
+    weight_by_size: true
+metadata:
+  version: 0.0

lm_eval/tasks/paws-x/pawsx_template_yaml

 # This file will be included in the generated language-specific task configs.
 # It doesn't have a yaml file extension as it is not meant to be imported directly
 # by the harness.
-group: pawsx
 task: null
 dataset_path: paws-x
 dataset_name: null

lm_eval/tasks/pile/pile_arxiv.yaml

-group:
-  - pile
 task: pile_arxiv
 dataset_path: EleutherAI/pile
 dataset_name: pile_arxiv

lm_eval/tasks/polemo2/polemo2_in.yaml

-group:
+tag:
   - polemo2
 task: polemo2_in
 dataset_path: allegro/klej-polemo2-in

lm_eval/tasks/qa4mre/qa4mre_2011.yaml

-group:
+tag:
   - qa4mre
 task: qa4mre_2011
 dataset_path: qa4mre

lm_eval/tasks/qasper/bool.yaml

-group: qasper
+tag: qasper
 task: qasper_bool
 dataset_path: allenai/qasper
 output_type: multiple_choice

lm_eval/tasks/qasper/freeform.yaml

-group: qasper
+tag: qasper
 task: qasper_freeform
 dataset_path: allenai/qasper
 output_type: generate_until

lm_eval/tasks/scrolls/task.py

@@ -116,7 +116,7 @@ class _SCROLLSTask(ConfigurableTask):
     PRUNE_MAX_TOKENS = None
     PRUNE_NUM_PROC = None
 
-    def __init__(self):
+    def __init__(self, config=None):
         super().__init__(config={"metadata": {"version": self.VERSION}})
         if self.DATASET_NAME is not None:
             self.metric = load_metric(_download_metric(), config_name=self.DATASET_NAME)

lm_eval/tasks/squad_completion/task.py

@@ -12,7 +12,7 @@ class SQUADCompletion(ConfigurableTask):
     DATASET_PATH = "hazyresearch/based-squad"
     DATASET_NAME = "default"
 
-    def __init__(self):
+    def __init__(self, **kwargs):
         super().__init__(config={"metadata": {"version": self.VERSION}})
 
     def has_training_docs(self):

lm_eval/tasks/squadv2/task.py

@@ -52,7 +52,7 @@ class SQuAD2(ConfigurableTask):
     DATASET_PATH = "squad_v2"
     DATASET_NAME = None
 
-    def __init__(self):
+    def __init__(self, config=None):
         super().__init__(config={"metadata": {"version": self.VERSION}})
 
     # HF changed squad on us so we have to make sure we aren't running the old one

lm_eval/tasks/storycloze/storycloze_2016.yaml

-group: storycloze
+tag: storycloze
 task: storycloze_2016
 dataset_path: story_cloze
 dataset_name: 2016

lm_eval/tasks/storycloze/storycloze_2018.yaml

-group: storycloze
+tag: storycloze
 task: storycloze_2018
 dataset_path: story_cloze
 dataset_name: 2018

lm_eval/tasks/super_glue/README.md

@@ -26,10 +26,14 @@ Homepage: https://super.gluebenchmark.com/
 }
 ```
 
-### Groups and Tasks
+### Groups, Tags, and Tasks
 
 #### Groups
 
+None.
+
+#### Tags
+
 * `super-glue-lm-eval-v1`: SuperGLUE eval adapted from LM Eval V1
 * `super-glue-t5-prompt`: SuperGLUE prompt and evaluation that matches the T5 paper (if using accelerate, will error if record is included.)