reformat

lm_eval/datasets/model_written_evals/advanced_ai_risk.py

@@ -74,18 +74,23 @@ split_alias = {
 }
 
 _URLS = {
-    f"{split_alias[split]}-{subset}": f"https://raw.githubusercontent.com/anthropics/evals/main/advanced-ai-risk/{split}/{subset}.jsonl" for split in split_name for subset in subset_names
+    f"{split_alias[split]}-{subset}": f"https://raw.githubusercontent.com/anthropics/evals/main/advanced-ai-risk/{split}/{subset}.jsonl"
+    for split in split_name
+    for subset in subset_names
 }
 _URLS.pop("human-self-awareness-training-nn-architecture")
 _URLS.pop("fewshot-self-awareness-training-nn-architecture")
 
+
 class AdvancedAIRisk(datasets.GeneratorBasedBuilder):
     """TODO: Short description of my dataset."""
 
     VERSION = datasets.Version("1.0.0")
 
     BUILDER_CONFIGS = [
-            datasets.BuilderConfig(name=f"{split_alias[split]}-{subset}") for split in split_name for subset in subset_names
+        datasets.BuilderConfig(name=f"{split_alias[split]}-{subset}")
+        for split in split_name
+        for subset in subset_names
     ]
 
     def _info(self):

lm_eval/datasets/model_written_evals/persona.py

@@ -177,11 +177,12 @@ subset_names = [
     "okay-with-deceiving-a-user-to-help-them",
     "desire-for-computational-efficiency",
     "low-discount-rate",
-    "desire-for-no-human-oversight-sometimes"
+    "desire-for-no-human-oversight-sometimes",
 ]
 
 _URLS = {
-    subset: f"https://raw.githubusercontent.com/anthropics/evals/main/persona/{subset}.jsonl" for subset in subset_names
+    subset: f"https://raw.githubusercontent.com/anthropics/evals/main/persona/{subset}.jsonl"
+    for subset in subset_names
 }
 
 
@@ -190,9 +191,7 @@ class Persona(datasets.GeneratorBasedBuilder):
 
     VERSION = datasets.Version("1.0.0")
 
-    BUILDER_CONFIGS = [
-            datasets.BuilderConfig(name=subset) for subset in subset_names
-    ]
+    BUILDER_CONFIGS = [datasets.BuilderConfig(name=subset) for subset in subset_names]
 
     def _info(self):
         features = datasets.Features(

lm_eval/datasets/model_written_evals/sycophancy.py

@@ -44,11 +44,12 @@ _LICENSE = "CC-BY-4.0 license"
 subset_names = [
     "sycophancy_on_nlp_survey",
     "sycophancy_on_philpapers2020",
-    "sycophancy_on_political_typology_quiz"
+    "sycophancy_on_political_typology_quiz",
 ]
 
 _URLS = {
-    subset: f"https://raw.githubusercontent.com/anthropics/evals/main/sycophancy/{subset}.jsonl" for subset in subset_names
+    subset: f"https://raw.githubusercontent.com/anthropics/evals/main/sycophancy/{subset}.jsonl"
+    for subset in subset_names
 }
 
 
@@ -57,21 +58,20 @@ class Sycophancy(datasets.GeneratorBasedBuilder):
 
     VERSION = datasets.Version("1.0.0")
 
-    BUILDER_CONFIGS = [
-            datasets.BuilderConfig(name=subset) for subset in subset_names
-    ]
+    BUILDER_CONFIGS = [datasets.BuilderConfig(name=subset) for subset in subset_names]
 
     def _info(self):
         feature_dict = {
-                "question": datasets.Value("string"),
-                "answer_matching_behavior": datasets.Value("string"),
-                "answer_not_matching_behavior": datasets.Value("string"),
-            }
+            "question": datasets.Value("string"),
+            "answer_matching_behavior": datasets.Value("string"),
+            "answer_not_matching_behavior": datasets.Value("string"),
+        }
         if self.config.name == "sycophancy_on_political_typology_quiz":
             feature_dict = {
                 **feature_dict,
-                **{"user_affiliation": datasets.Value("string"),
-                }
+                **{
+                    "user_affiliation": datasets.Value("string"),
+                },
             }
         features = datasets.Features(feature_dict)
 

lm_eval/datasets/model_written_evals/winogenerated.py

@@ -47,7 +47,8 @@ subset_names = [
 ]
 
 _URLS = {
-    subset: f"https://raw.githubusercontent.com/anthropics/evals/main/winogenerated/{subset}.jsonl" for subset in subset_names
+    subset: f"https://raw.githubusercontent.com/anthropics/evals/main/winogenerated/{subset}.jsonl"
+    for subset in subset_names
 }
 
 
@@ -56,9 +57,7 @@ class Winogenerated(datasets.GeneratorBasedBuilder):
 
     VERSION = datasets.Version("1.0.0")
 
-    BUILDER_CONFIGS = [
-            datasets.BuilderConfig(name=subset) for subset in subset_names
-    ]
+    BUILDER_CONFIGS = [datasets.BuilderConfig(name=subset) for subset in subset_names]
 
     def _info(self):
         if self.config.name == "winogenerated_examples":