Merge branch 'EleutherAI:main' into mmlu-pro

lm_eval/tasks/paloma/paloma_m2d2_wikipedia_unsplit.yaml

+include: _paloma_template
+task: paloma_m2d2_wikipedia_unsplit
+task_alias: M2D2 Wikipedia
+dataset_name: m2d2_wikipedia_unsplit

lm_eval/tasks/paloma/paloma_manosphere_meta_sep.yaml

+include: _paloma_template
+task: paloma_manosphere_meta_sep
+task_alias: Manosphere
+dataset_name: manosphere_meta_sep

lm_eval/tasks/paloma/paloma_mc4.yaml

+include: _paloma_template
+task: paloma_mc4
+task_alias: mC4
+dataset_name: mc4

lm_eval/tasks/paloma/paloma_ptb.yaml

+include: _paloma_template
+task: paloma_ptb
+task_alias: PTB
+dataset_name: ptb

lm_eval/tasks/paloma/paloma_redpajama.yaml

+include: _paloma_template
+task: paloma_redpajama
+task_alias: RedPajama
+dataset_name: redpajama

lm_eval/tasks/paloma/paloma_twitterAAE_HELM_fixed.yaml

+include: _paloma_template
+task: paloma_twitterAAE_HELM_fixed
+task_alias: Twitter AAE
+dataset_name: twitterAAE_HELM_fixed

lm_eval/tasks/paloma/paloma_utils.py

+def doc_to_target(doc):
+    return str(doc["text"])

lm_eval/tasks/paloma/paloma_wikitext_103.yaml

+include: _paloma_template
+task: paloma_wikitext_103
+task_alias: Wikitext-103
+dataset_name: wikitext_103

lm_eval/tasks/piqa/piqa.yaml

@@ -19,3 +19,5 @@ metric_list:
     higher_is_better: true
 metadata:
   version: 1.0
+dataset_kwargs:
+  trust_remote_code: true

scripts/write_out.py

@@ -70,6 +70,11 @@ def main():
             if docs is not None:
                 iters.append(docs)
 
+        if len(iters) == 0:
+            raise ValueError(
+                f"Passed --sets '{args.sets}' but this task has no splits which match. Please specify a different --sets value."
+            )
+
         docs = join_iters(iters)
 
         with open(