adjustments to have `task` to modify a task name in benchmark

lm_eval/tasks/__init__.py

@@ -47,16 +47,16 @@ def check_prompt_config(config):
             subset_name=config["dataset_name"] if "dataset_name" in config else None,
         )
         for idx, prompt_variation in enumerate(prompt_list):
-            task_name = [config["task"]] if "task" in config else []
             all_configs.append(
                 {
                     **config,
                     **{"use_prompt": prompt_variation},
                     **{
                         "task": "_".join(
-                            task_name
-                            + [
-                                get_task_name_from_config(config),
+                            [
+                                config["task"]
+                                if "task" in config
+                                else get_task_name_from_config(config),
                                 prompt_variation,
                             ]
                         )
@@ -100,58 +100,8 @@ def include_task_folder(task_dir):
                         )
 
 
-# def include_benchmarks(task_dir, benchmark_dir="benchmarks"):
-
-#     for root, subdirs, file_list in os.walk(os.path.join(task_dir, benchmark_dir)):
-#         if (subdirs == [] or subdirs == ["__pycache__"]) and (len(file_list) > 0):
-#             for f in file_list:
-#                 if f.endswith(".yaml"):
-#                     try:
-#                         benchmark_path = os.path.join(root, f)
-
-#                         with open(benchmark_path, "rb") as file:
-#                             yaml_config = yaml.full_load(file)
-
-#                         assert "group" in yaml_config
-#                         group = yaml_config["group"]
-#                         all_task_list = yaml_config["task"]
-#                         config_list = [
-#                             task for task in all_task_list if type(task) != str
-#                         ]
-#                         task_list = [
-#                             task for task in all_task_list if type(task) == str
-#                         ]
-
-#                         for task_config in config_list:
-#                             var_configs = check_prompt_config(
-#                                 {
-#                                     **task_config,
-#                                     **{"group": group},
-#                                 }
-#                             )
-#                             for config in var_configs:
-#                                 register_configurable_task(config)
-
-#                         task_names = utils.pattern_match(task_list, ALL_TASKS)
-#                         for task in task_names:
-#                             if task in TASK_REGISTRY:
-#                                 if group in GROUP_REGISTRY:
-#                                     GROUP_REGISTRY[group].append(task)
-#                                 else:
-#                                     GROUP_REGISTRY[group] = [task]
-#                                     ALL_TASKS.add(group)
-#                     except Exception as error:
-#                         eval_logger.warning(
-#                             "Failed to load benchmark in\n"
-#                             f"                                 {benchmark_path}\n"
-#                             "                                 Benchmark will not be added to registry\n"
-#                             f"                                 Error: {error}"
-#                         )
-
-
 task_dir = os.path.dirname(os.path.abspath(__file__)) + "/"
 include_task_folder(task_dir)
-# include_benchmarks(task_dir)
 
 
 def get_task(task_name, config):