reformat

lm_eval/evaluator.py

@@ -200,7 +200,10 @@ def evaluate(
         # aggregate Instances by LM method requested to get output.
         reqtype = (
             "loglikelihood"
-            if (task.OUTPUT_TYPE == "multiple_choice" or task.OUTPUT_TYPE == "winograd_schema") 
+            if (
+                task.OUTPUT_TYPE == "multiple_choice"
+                or task.OUTPUT_TYPE == "winograd_schema"
+            )
             else task.OUTPUT_TYPE
         )  # TODO: this is hacky, fix in task.py
         requests[reqtype].extend(task.instances)

lm_eval/tasks/pubmedqa/preprocess_pubmedqa.py

@@ -4,13 +4,11 @@ def doc_to_text(doc):
         ctxs, doc["question"], doc["final_decision"]
     )
 
+
 def doc_to_target(doc):
     return " {}".format(doc["final_decision"])
 
+
 def gold_alias(doc):
-    dict_to_label = {
-        'yes': 0,
-        'no': 1,
-        'maybe': 2
-    }
-    return dict_to_label[doc["final_decision"]]
+    dict_to_label = {"yes": 0, "no": 1, "maybe": 2}
+    return dict_to_label[doc["final_decision"]]
\ No newline at end of file

lm_eval/tasks/race/preprocess_race.py

-import ast 
+import ast
+
 
 def process_ast(string):
     return ast.literal_eval(string)
 
+
 def last_problem(doc):
     return process_ast(doc["problems"])[-1]
 
+
 def get_answer_option(problem):
     letter_to_num = {"A": 0, "B": 1, "C": 2, "D": 3}
     answer = letter_to_num[problem["answer"]]
     return problem["options"][answer]
 
+
 def create_choices(doc):
     problem = last_problem(doc)
     choices = [problem["options"][i] for i in range(4)]
     return choices
 
+
 def doc_to_text(doc):
     text = "Article: " + doc["article"] + "\n\n"
     for problem in process_ast(doc["problems"])[:-1]:
         if problem["question"][-6:] == "  _  .":
-            text += (
-                problem["question"][-5:] + get_answer_option(problem) + "\n"
-            )
+            text += problem["question"][-5:] + get_answer_option(problem) + "\n"
         else:
             question = "Question: " + problem["question"] + "\n"
             answer = "Answer: " + get_answer_option(problem) + "\n"
@@ -30,6 +33,7 @@ def doc_to_text(doc):
     text += last_problem(doc)["question"]
     return text
 
+
 def doc_to_target(doc):
     letter_to_num = {"A": 0, "B": 1, "C": 2, "D": 3}
     answer = letter_to_num[last_problem(doc)["answer"]]