evaluator: fix bleu/chrf/ter taking forever

lm_eval/evaluator.py

@@ -93,7 +93,9 @@ def evaluate(lm, task_dict, provide_description, num_fewshot, limit, bootstrap_i
         task = task_dict[task_name]
         results[task_name][metric] = task.aggregation()[metric](items)
 
-        stderr = lm_eval.metrics.stderr_for_metric(task.aggregation()[metric], bootstrap_iters=bootstrap_iters)
+        # hotfix: bleu, chrf, ter seem to be really expensive to bootstrap
+        # so we run them less iterations. still looking for a cleaner way to do this
+        stderr = lm_eval.metrics.stderr_for_metric(task.aggregation()[metric], bootstrap_iters=1000 if metric in ["bleu", "chrf", "ter"] else bootstrap_iters)
         if stderr is not None:
             results[task_name][metric + "_stderr"] = stderr(items)