formatted

lm_eval/tasks/pubmedqa/preprocess_pubmedqa.py

@@ -4,13 +4,11 @@ def doc_to_text(doc):
         ctxs, doc["question"], doc["final_decision"]
     )
 
+
 def doc_to_target(doc):
     return " {}".format(doc["final_decision"])
 
+
 def gold_alias(doc):
-    dict_to_label = {
-        'yes': 0,
-        'no': 1,
-        'maybe': 2
-    }
-    return dict_to_label[doc["final_decision"]]
+    dict_to_label = {"yes": 0, "no": 1, "maybe": 2}
+    return dict_to_label[doc["final_decision"]]
\ No newline at end of file

lm_eval/tasks/pubmedqa/pubmedqa.yaml

@@ -14,4 +14,4 @@ gold_alias: !function preprocess_pubmedqa.gold_alias
 metric_list:
   - metric: acc
     aggregation: mean
-    higher_is_better: true
+    higher_is_better: true
\ No newline at end of file

lm_eval/tasks/race/preprocess_race.py

-import ast 
+import ast
+
 
 def process_ast(string):
     return ast.literal_eval(string)
 
+
 def last_problem(doc):
     return process_ast(doc["problems"])[-1]
 
+
 def get_answer_option(problem):
     letter_to_num = {"A": 0, "B": 1, "C": 2, "D": 3}
     answer = letter_to_num[problem["answer"]]
     return problem["options"][answer]
 
+
 def create_choices(doc):
     problem = last_problem(doc)
     choices = [problem["options"][i] for i in range(4)]
     return choices
 
+
 def doc_to_text(doc):
     text = "Article: " + doc["article"] + "\n\n"
     for problem in process_ast(doc["problems"])[:-1]:
         if problem["question"][-6:] == "  _  .":
-            text += (
-                problem["question"][-5:] + get_answer_option(problem) + "\n"
-            )
+            text += problem["question"][-5:] + get_answer_option(problem) + "\n"
         else:
             question = "Question: " + problem["question"] + "\n"
             answer = "Answer: " + get_answer_option(problem) + "\n"
@@ -30,6 +33,7 @@ def doc_to_text(doc):
     text += last_problem(doc)["question"]
     return text
 
+
 def doc_to_target(doc):
     letter_to_num = {"A": 0, "B": 1, "C": 2, "D": 3}
     answer = letter_to_num[last_problem(doc)["answer"]]

lm_eval/tasks/swag/swag.yaml

@@ -17,4 +17,4 @@ metric_list:
     higher_is_better: true
   - metric: acc_norm
     aggregation: mean
-    higher_is_better: true
+    higher_is_better: true
\ No newline at end of file

lm_eval/tasks/winogrande/preprocess_winogrande.py

@@ -4,11 +4,13 @@ def partial_context(doc, option):
     pronoun_loc = doc["sentence"].index("_")
     return doc["sentence"][:pronoun_loc] + option
 
+
 def partial_target(doc):
     # The target is everything after the document specified pronoun.
     pronoun_loc = doc["sentence"].index("_") + 1
     return doc["sentence"][pronoun_loc:].strip()
 
+
 def create_choices(doc):
     choices = []
     for option in [doc["option1"], doc["option2"]]:
@@ -16,6 +18,7 @@ def create_choices(doc):
         choices.append(partial_ctx)
     return choices
 
+
 def gold_alias(doc):
     answer_to_num = {"1": 0, "2": 1}
-    return answer_to_num[doc['answer']]
+    return answer_to_num[doc["answer"]]
\ No newline at end of file