add draft abstract algebra task, fix genning script

.pre-commit-config.yaml

@@ -40,6 +40,6 @@ repos:
       - id: codespell
         exclude: >
           (?x)^(
-              .*\.json|ignore.txt
+              .*\.json|ignore.txt|.*\.yaml
           )$
         args: [--check-filenames, --check-hidden, --ignore-words=ignore.txt]

lm_eval/tasks/mmlu/gen_all_splits.py

@@ -97,6 +97,7 @@ if __name__ == "__main__":
             "include": base_yaml_name,
             "task": base_yaml["task"].strip("abstract_algebra") + "subject",
             "dataset_name": subject,
+            "description": f"The following are multiple choice questions (with answers) about {' '.join(subject.split('_'))}.\n\n",
         }
 
         file_save_path = args.task_save_path + f"_{subject}.yaml"

lm_eval/tasks/mmlu/hendrycks_test_original_default.yaml

@@ -2,16 +2,16 @@ group:
   - mmlu
   - mmlu_original
   - multiple_choice
-task: mmlu_original_
+task: mmlu_original_abstract_algebra
 dataset_path: cais/mmlu
 dataset_name: abstract_algebra
 output_type: multiple_choice
-training_split: train
 validation_split: validation
 test_split: test
-doc_to_text: "Question: {{question}}\nAnswer:"
-doc_to_target: "{{choices.label.index(answerKey)}}"
-doc_to_choice: "{{choices.text}}"
+description: "The following are multiple choice questions (with answers) about abstract algebra.\n\n"
+doc_to_text: "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:"
+doc_to_choice: "{{choices}}"
+doc_to_target: "{{answer}}"
 metric_list:
   - metric: acc
     aggregation: mean