reformat

lm_eval/api/metrics.py

@@ -127,10 +127,10 @@ def brier_score(items):  # This is a passthrough function
         _p = np.array(p)
         _g = np.array(g)
         _g_one_hot = np.eye(len(_p[0]))[_g]
-        average += np.mean(np.sum((_p - _g_one_hot) ** 2, axis=1))*len(_g)
+        average += np.mean(np.sum((_p - _g_one_hot) ** 2, axis=1)) * len(_g)
         total_size += len(_g)
 
-    return average/total_size
+    return average / total_size
 
 
 @register_metric(

lm_eval/api/task.py

@@ -1020,7 +1020,7 @@ class ConfigurableTask(Task):
                 **({"perplexity": ll} if "perplexity" in use_metric else {}),
                 **({"acc": int(is_greedy)} if "acc" in use_metric else {}),
                 **(
-                    {"brier_score": (0, [prob_norm])} # Gold is Index 0
+                    {"brier_score": (0, [prob_norm])}  # Gold is Index 0
                     if "brier_score" in use_metric
                     else {}
                 ),

lm_eval/tasks/arc/alternative_worlds/README.md

@@ -15,6 +15,6 @@ Answer types:
     - original option
     - just letter
 - letters + continuation
-    - original option 
+    - original option
     - just letter
-- continuation
+- continuation
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/style_01/a.yaml

 include: ../_arc_easy_alt_yaml
 group: arc_easy_01
-group_alias: style_01 
+group_alias: style_01
 task: arc_easy_01a
 task_alias: a
 doc_to_text: !function ../styles.template_01
-doc_to_choice: !function ../styles.choice_01a
+doc_to_choice: !function ../styles.choice_01a
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/style_01/b.yaml

 include: ../_arc_easy_alt_yaml
 group: arc_easy_01
-group_alias: style_01 
+group_alias: style_01
 task: arc_easy_01b
 task_alias: b
 doc_to_text: !function ../styles.template_01
-doc_to_choice: !function ../styles.choice_01b
+doc_to_choice: !function ../styles.choice_01b
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/style_01/c.yaml

 include: ../_arc_easy_alt_yaml
 group: arc_easy_01
-group_alias: style_01 
+group_alias: style_01
 task: arc_easy_01c
 task_alias: c
 doc_to_text: !function ../styles.template_01
-doc_to_choice: !function ../styles.choice_01c
+doc_to_choice: !function ../styles.choice_01c
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/style_02/a.yaml

 include: ../_arc_easy_alt_yaml
 group: arc_easy_02
-group_alias: style_02 
+group_alias: style_02
 task: arc_easy_02a
 task_alias: a
 doc_to_text: !function ../styles.template_02
-doc_to_choice: !function ../styles.choice_02a
+doc_to_choice: !function ../styles.choice_02a
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/style_02/b.yaml

 include: ../_arc_easy_alt_yaml
 group: arc_easy_02
-group_alias: style_02 
+group_alias: style_02
 task: arc_easy_02b
 task_alias: b
 doc_to_text: !function ../styles.template_02
-doc_to_choice: !function ../styles.choice_02b
+doc_to_choice: !function ../styles.choice_02b
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/style_02/c.yaml

 include: ../_arc_easy_alt_yaml
 group: arc_easy_02
-group_alias: style_02 
+group_alias: style_02
 task: arc_easy_02c
 task_alias: c
 doc_to_text: !function ../styles.template_02
-doc_to_choice: !function ../styles.choice_02c
+doc_to_choice: !function ../styles.choice_02c
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/style_03/a.yaml

 include: ../_arc_easy_alt_yaml
 group: arc_easy_03
-group_alias: style_03 
+group_alias: style_03
 task: arc_easy_03a
 task_alias: a
 doc_to_text: !function ../styles.template_03
-doc_to_choice: !function ../styles.choice_03a
+doc_to_choice: !function ../styles.choice_03a
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/style_03/b.yaml

 include: ../_arc_easy_alt_yaml
 group: arc_easy_03
-group_alias: style_03 
+group_alias: style_03
 task: arc_easy_03b
 task_alias: b
 doc_to_text: !function ../styles.template_03
-doc_to_choice: !function ../styles.choice_03b
+doc_to_choice: !function ../styles.choice_03b
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/style_03/c.yaml

 include: ../_arc_easy_alt_yaml
 group: arc_easy_03
-group_alias: style_03 
+group_alias: style_03
 task: arc_easy_03c
 task_alias: c
 doc_to_text: !function ../styles.template_03
-doc_to_choice: !function ../styles.choice_03c
+doc_to_choice: !function ../styles.choice_03c
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/style_04/a.yaml

 include: ../_arc_easy_alt_yaml
 group: arc_easy_04
-group_alias: style_04 
+group_alias: style_04
 task: arc_easy_04a
 task_alias: a
 doc_to_text: !function ../styles.template_04
-doc_to_choice: !function ../styles.choice_04a
+doc_to_choice: !function ../styles.choice_04a
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/style_04/b.yaml

 include: ../_arc_easy_alt_yaml
 group: arc_easy_04
-group_alias: style_04 
+group_alias: style_04
 task: arc_easy_04b
 task_alias: b
 doc_to_text: !function ../styles.template_04
-doc_to_choice: !function ../styles.choice_04b
+doc_to_choice: !function ../styles.choice_04b
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/style_04/c.yaml

 include: ../_arc_easy_alt_yaml
 group: arc_easy_04
-group_alias: style_04 
+group_alias: style_04
 task: arc_easy_04c
 task_alias: c
 doc_to_text: !function ../styles.template_04
-doc_to_choice: !function ../styles.choice_04c
+doc_to_choice: !function ../styles.choice_04c
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/style_05/a.yaml

 include: ../_arc_easy_alt_yaml
 group: arc_easy_05
-group_alias: style_05 
+group_alias: style_05
 task: arc_easy_05a
 task_alias: a
 doc_to_text: !function ../styles.template_05
-doc_to_choice: !function ../styles.choice_05a
+doc_to_choice: !function ../styles.choice_05a
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/style_05/b.yaml

 include: ../_arc_easy_alt_yaml
 group: arc_easy_05
-group_alias: style_05 
+group_alias: style_05
 task: arc_easy_05b
 task_alias: b
 doc_to_text: !function ../styles.template_05
-doc_to_choice: !function ../styles.choice_05b
+doc_to_choice: !function ../styles.choice_05b
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/style_05/c.yaml

 include: ../_arc_easy_alt_yaml
 group: arc_easy_05
-group_alias: style_05 
+group_alias: style_05
 task: arc_easy_05c
 task_alias: c
 doc_to_text: !function ../styles.template_05
-doc_to_choice: !function ../styles.choice_05c
+doc_to_choice: !function ../styles.choice_05c
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/style_06/a.yaml

 include: ../_arc_easy_alt_yaml
 group: arc_easy_06
-group_alias: style_06 
+group_alias: style_06
 task: arc_easy_06a
 task_alias: a
 doc_to_text: !function ../styles.template_06
-doc_to_choice: !function ../styles.choice_06a
+doc_to_choice: !function ../styles.choice_06a
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/style_06/b.yaml

 include: ../_arc_easy_alt_yaml
 group: arc_easy_06
-group_alias: style_06 
+group_alias: style_06
 task: arc_easy_06b
 task_alias: b
 doc_to_text: !function ../styles.template_06
-doc_to_choice: !function ../styles.choice_06b
+doc_to_choice: !function ../styles.choice_06b
\ No newline at end of file