Merge branch 'main' of https://github.com/EleutherAI/lm-evaluation-harness into alt_worlds

lm_eval/tasks/glue/mnli/mismatch.yaml

 include: default.yaml
 task: mnli_mismatch
 validation_split: validation_mismatched
-test_split: test_mismatched

lm_eval/tasks/glue/qqp/default.yaml

@@ -5,7 +5,6 @@ dataset_name: qqp
 output_type: multiple_choice
 training_split: train
 validation_split: validation
-test_split: test
 doc_to_text: "\nSentence 1: {{question1}}\nSentence 2: {{question2}}\nAnswer:"
 doc_to_target: label
 doc_to_choice: ["no", "yes"]

lm_eval/tasks/hellaswag/hellaswag.yaml

@@ -10,7 +10,7 @@ test_split: null
 process_docs: !function utils.process_docs
 doc_to_text: "{{query}}"
 doc_to_target: "{{label}}"
-doc_to_choice: "{{choices}}"
+doc_to_choice: "choices"
 metric_list:
   - metric: acc
     aggregation: mean

scripts/write_out.py

@@ -12,7 +12,7 @@ EXAMPLE_DIVIDER = "!!@@##@@!! -- Example {i}\n"
 
 def parse_args():
     parser = argparse.ArgumentParser()
-    parser.add_argument("--output_base_path", required=True)
+    parser.add_argument("--output_base_path", "--output_path", required=True)
     parser.add_argument("--tasks", default="all_tasks")
     parser.add_argument("--sets", type=str, default="val")  # example: val,test
     parser.add_argument("--num_fewshot", type=int, default=1)
@@ -51,6 +51,8 @@ def main():
 
     os.makedirs(args.output_base_path, exist_ok=True)
     for task_name, task in task_dict.items():
+        if type(task) == tuple:
+            group_name, task = task
         rnd = random.Random()
         rnd.seed(args.seed)