reformat

lm_eval/api/task.py

@@ -91,7 +91,9 @@ class TaskConfig(dict):
     should_decontaminate: bool = False
     doc_to_decontamination_query: str = None
 
-    metadata: Union[str, list] = None # by default, not used in the code. allows for users to pass arbitrary info to tasks
+    metadata: Union[
+        str, list
+    ] = None  # by default, not used in the code. allows for users to pass arbitrary info to tasks
 
     def __post_init__(self) -> None:
         if self.dataset_path and ("." in self.dataset_path):

lm_eval/tasks/anli/anli_r1.yaml

@@ -23,4 +23,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
\ No newline at end of file
+  - version: 1.0

lm_eval/tasks/arc/arc_easy.yaml

@@ -20,4 +20,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
\ No newline at end of file
+  - version: 1.0

lm_eval/tasks/arithmetic/arithmetic_1dc.yaml

@@ -13,4 +13,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
\ No newline at end of file
+  - version: 1.0

lm_eval/tasks/asdiv/default.yaml

@@ -11,4 +11,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
\ No newline at end of file
+  - version: 1.0

lm_eval/tasks/babi/babi.yaml

@@ -17,4 +17,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 0.0
\ No newline at end of file
+  - version: 0.0

lm_eval/tasks/bbh/cot_fewshot/_cot_fewshot_template_yaml

@@ -25,4 +25,4 @@ filter_list:
         regex_pattern: "(?<=the answer is )(.*)(?=.)"
       - function: "take_first"
 metadata:
-  - version: 0.0
\ No newline at end of file
+  - version: 0.0

lm_eval/tasks/bbh/cot_zeroshot/_cot_zeroshot_template_yaml

@@ -23,4 +23,4 @@ filter_list:
         regex_pattern: "((?<=The answer is )(.*)(?=.)|(?<=the answer is )(.*)(?=.)|(?<=The answer: )(.*)(?=.)|(?<=The final answer: )(.*)(?=.))"
       - function: "take_first"
 metadata:
-  - version: 0
\ No newline at end of file
+  - version: 0

lm_eval/tasks/bbh/fewshot/_fewshot_template_yaml

@@ -17,4 +17,4 @@ generation_kwargs:
   do_sample: false
   temperature: 0.0
 metadata:
-  - version: 0
\ No newline at end of file
+  - version: 0

lm_eval/tasks/bbh/zeroshot/_zeroshot_template_yaml

@@ -17,4 +17,4 @@ generation_kwargs:
   do_sample: false
   temperature: 0.0
 metadata:
-  - version: 0
\ No newline at end of file
+  - version: 0

lm_eval/tasks/belebele/_default_template_yaml

@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 0.0
\ No newline at end of file
+  - version: 0.0

lm_eval/tasks/bigbench/generate_until_template_yaml

@@ -15,4 +15,4 @@ metric_list:
     higher_is_better: true
     ignore_punctuation: true
 metadata:
-  - version: 0.0
\ No newline at end of file
+  - version: 0.0

lm_eval/tasks/bigbench/multiple_choice_template_yaml

@@ -12,4 +12,4 @@ metric_list:
   - metric: acc
   # TODO: brier score and other metrics
 metadata:
-  - version: 0.0
\ No newline at end of file
+  - version: 0.0

lm_eval/tasks/blimp/template_yaml

@@ -10,4 +10,4 @@ doc_to_decontamination_query: "{{sentence_good}} {{sentence_bad}}"
 metric_list:
   - metric: acc
 metadata:
-  - version: 1.0
\ No newline at end of file
+  - version: 1.0

lm_eval/tasks/cmmlu/_default_template_yaml

@@ -16,4 +16,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 0.0
\ No newline at end of file
+  - version: 0.0

lm_eval/tasks/code_x_glue/code-text/go.yaml

@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: True
 metadata:
-  - version: 0.0
\ No newline at end of file
+  - version: 0.0

lm_eval/tasks/code_x_glue/code-text/java.yaml

@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: True
 metadata:
-  - version: 0.0
\ No newline at end of file
+  - version: 0.0

lm_eval/tasks/code_x_glue/code-text/javascript.yaml

@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: True
 metadata:
-  - version: 0.0
\ No newline at end of file
+  - version: 0.0

lm_eval/tasks/code_x_glue/code-text/php.yaml

@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: True
 metadata:
-  - version: 0.0
\ No newline at end of file
+  - version: 0.0

lm_eval/tasks/code_x_glue/code-text/python.yaml

@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: True
 metadata:
-  - version: 0.0
\ No newline at end of file
+  - version: 0.0