removed delimiter=""

lm_eval/__main__.py

@@ -97,6 +97,12 @@ def parse_eval_args() -> argparse.Namespace:
         default=None,
         help="Additional path to include if there are external tasks to include.",
     )
+    parser.add_argument(
+        "--verbose",
+        type=bool,
+        default=False,
+        help="Log error when tasks are not registered.",
+    )
     return parser.parse_args()
 
 
@@ -167,6 +173,7 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:
         assert args.output_path, "Specify --output_path"
 
     eval_logger.info(f"Selected Tasks: {task_names}")
+    eval_logger.verbose = args.verbose
 
     results = evaluator.simple_evaluate(
         model=args.model,

lm_eval/tasks/__init__.py

@@ -139,15 +139,18 @@ def include_task_folder(task_dir: str, register_task: bool = True) -> None:
                                 register_configurable_group(config, yaml_path)
 
                 except Exception as error:
-                    import traceback
-
-                    eval_logger.warning(
-                        "Failed to load config in\n"
-                        f"                                 {yaml_path}\n"
-                        "                                 Config will not be added to registry\n"
-                        f"                                 Error: {error}\n"
-                        f"                                 Traceback: {traceback.format_exc()}"
-                    )
+                    if eval_logger.verbose:
+                        import traceback
+
+                        eval_logger.warning(
+                            "Failed to load config in\n"
+                            f"                                 {yaml_path}\n"
+                            "                                 Config will not be added to registry\n"
+                            f"                                 Error: {error}\n"
+                            f"                                 Traceback: {traceback.format_exc()}"
+                        )
+                    else:
+                        eval_logger.warning("Yaml failed to register {yaml_path}\n")
     return 0
 
 

lm_eval/tasks/model_written_evals/persona/_template_yaml

@@ -2,7 +2,6 @@ group: persona
 dataset_path: EleutherAI/persona
 output_type: multiple_choice
 validation_split: validation
-target_delimiter: ""
 doc_to_text: "{{question}}"
 doc_to_target: 0
 doc_to_choice: "{{[answer_matching_behavior, answer_not_matching_behavior]}}"