Merge pull request #1002 from EleutherAI/bugfixes-bigrefactor

[Refactor] Bugfixes

lm_eval/__main__.py

@@ -149,7 +149,11 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:
                 if os.path.isfile(task):
                     config = utils.load_yaml_config(task)
                     task_names.append(config)
-            task_missing = [task for task in tasks_list if task not in task_names]
+            task_missing = [
+                task
+                for task in tasks_list
+                if task not in task_names and "*" not in task
+            ]  # we don't want errors if a wildcard ("*") task name was used
 
             if task_missing:
                 missing = ", ".join(task_missing)

lm_eval/tasks/bigbench/generate_until_template_yaml

-group: bigbench
-dataset_path: bigbench # will switch to `hails/bigbench` when all tasks are pushed
+group: bigbench_generate_until
+dataset_path: hails/bigbench
 output_type: generate_until
 dataset_kwargs:
   # num_shots: 0 # TODO: num of shots for `bigbench` HF dataset should be controlled through this, not through the typical methods

lm_eval/tasks/bigbench/multiple_choice_template_yaml

-group: bigbench
-dataset_path: bigbench # will switch to `hails/bigbench` when all tasks are pushed
+group: bigbench_multiple_choice
+dataset_path: hails/bigbench
 dataset_kwargs:
   # num_shots: 0 # TODO: num of shots for `bigbench` HF dataset should be controlled through this, not through the typical methods
   # subtask_name: null