make RougeScorer only initialized once (#2090)

lm_eval/tasks/tinyBenchmarks/utils_truthfulqa.py

@@ -6,6 +6,8 @@ from rouge_score import rouge_scorer, scoring
 
 """ This code mirrors the utils of the original truthful_qa task """
 
+ROUGE_SCORER = None
+
 
 def process_results_mc2(doc, results):
     lls, is_greedy = zip(*results)
@@ -153,7 +155,11 @@ def rouge(refs, preds):
         A `list` of predicted `strs`.
     """
     rouge_types = ["rouge1", "rouge2", "rougeLsum"]
-    scorer = rouge_scorer.RougeScorer(rouge_types)
+    global ROUGE_SCORER
+    if ROUGE_SCORER is None:
+        # init RougeScorer once (https://github.com/EleutherAI/lm-evaluation-harness/issues/1692)--rouge_types are constant
+        ROUGE_SCORER = rouge_scorer.RougeScorer(rouge_types)
+    scorer = ROUGE_SCORER
     # Add newlines between sentences to correctly compute `rougeLsum`.
 
     def _prepare_summary(summary):

lm_eval/tasks/truthfulqa/utils.py

@@ -4,6 +4,9 @@ import sacrebleu
 from rouge_score import rouge_scorer, scoring
 
 
+ROUGE_SCORER = None
+
+
 def process_results_mc2(doc, results):
     lls, is_greedy = zip(*results)
 
@@ -149,8 +152,14 @@ def rouge(refs, preds):
     :param preds:
         A `list` of predicted `strs`.
     """
+
     rouge_types = ["rouge1", "rouge2", "rougeLsum"]
-    scorer = rouge_scorer.RougeScorer(rouge_types)
+
+    global ROUGE_SCORER
+    if ROUGE_SCORER is None:
+        # init RougeScorer once (https://github.com/EleutherAI/lm-evaluation-harness/issues/1692)--rouge_types are constant
+        ROUGE_SCORER = rouge_scorer.RougeScorer(rouge_types)
+    scorer = ROUGE_SCORER
     # Add newlines between sentences to correctly compute `rougeLsum`.
 
     def _prepare_summary(summary):