add race task, with custom HF dataset

lm_eval/tasks/race/preprocess_race.py

+def last_problem(doc):
+    return doc["problems"][-1]
+
+def get_answer_option(problem):
+    letter_to_num = {"A": 0, "B": 1, "C": 2, "D": 3}
+    answer = letter_to_num[problem["answer"]]
+    return problem["options"][answer]
+
+def create_choices(doc):
+    problem = last_problem(doc)
+    choices = [problem["options"][i] for i in range(4)]
+    return choices
+
+def doc_to_text(doc):
+    text = "Article: " + doc["article"] + "\n\n"
+    for problem in doc["problems"][:-1]:
+        if problem["question"][-6:] == "  _  .":
+            text += (
+                problem["question"][-5:] + get_answer_option(problem) + "\n"
+            )
+        else:
+            question = "Question: " + problem["question"] + "\n"
+            answer = "Answer: " + get_answer_option(problem) + "\n"
+            text += question + answer
+    text += last_problem(doc)["question"]
+    return text
+
+def doc_to_target(doc):
+    letter_to_num = {"A": 0, "B": 1, "C": 2, "D": 3}
+    answer = letter_to_num[last_problem(doc)["answer"]]
+    return answer

lm_eval/tasks/race/race.yaml

+group:
+  - multiple_choice
+task: race
+dataset_path: bfattori/race_grouped
+dataset_name: high
+output_type: multiple_choice
+training_split: train
+validation_split: validation
+test_split: test
+create_choices: !function preprocess_race.create_choices
+doc_to_text: !function preprocess_race.doc_to_text
+doc_to_target: !function preprocess_race.doc_to_target
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
\ No newline at end of file