Optimization for evalita-llm rouge computation (#2878)

* feat: initial commit with templates for evalita evaluation

* fix: change rule for generate_until

* feat: modified yaml to use reduced version of NER test datasets

* feat: added templates to use reduced dataset for summarization (fanpage and ilpost)

* Add Six Prompts for Each Multiple-Choice Task

* fix: fastest eval for summarization

* chore: linted with ruff

* chore: linted with ruff

---------
Co-authored-by: rzanoli <zanoli@fbk.eu>

lm_eval/tasks/evalita_llm/_evalita-mp_sum_fp-small_p1.yaml

@@ -5,7 +5,7 @@ task_alias: prompt-1
 #doc_to_text: >
 #  "Crea un sommario del seguente testo. Testo: {{source}}\nSommario: "
 doc_to_text: "Riassumi il seguente articolo di giornale: '{{source}}'\nRiassunto:"
-process_results: !function utils.process_results_sum
+process_results: !function sum_utils.process_results_sum
 metric_list:
   - metric: rouge1
     higher_is_better: true

lm_eval/tasks/evalita_llm/_evalita-mp_sum_fp-small_p2.yaml

@@ -5,7 +5,7 @@ task_alias: prompt-2
 #doc_to_text: >
 #  "Crea un sommario del seguente testo. Testo: {{source}}\nSommario: "
 doc_to_text: "Devi risolvere un compito di sintesi automatica del testo. Riassumi il seguente articolo di giornale: '{{source}}'\nRiassunto:"
-process_results: !function utils.process_results_sum
+process_results: !function sum_utils.process_results_sum
 metric_list:
   - metric: rouge1
     higher_is_better: true

lm_eval/tasks/evalita_llm/_evalita-mp_sum_fp_p1.yaml

@@ -3,7 +3,7 @@ include: _sum_template_fp_yaml
 task: evalita-sp_sum_task_fp_p1
 task_alias: prompt-1
 doc_to_text: "Riassumi il seguente articolo di giornale: '{{source}}'\nRiassunto:"
-process_results: !function utils.process_results_sum
+process_results: !function sum_utils.process_results_sum
 metric_list:
   - metric: rouge1
     higher_is_better: true

lm_eval/tasks/evalita_llm/_evalita-mp_sum_fp_p2.yaml

@@ -3,7 +3,7 @@ include: _sum_template_fp_yaml
 task: evalita-sp_sum_task_fp_p2
 task_alias: prompt-2
 doc_to_text: "Devi risolvere un compito di sintesi automatica del testo. Riassumi il seguente articolo di giornale: '{{source}}'\nRiassunto:"
-process_results: !function utils.process_results_sum
+process_results: !function sum_utils.process_results_sum
 metric_list:
   - metric: rouge1
     higher_is_better: true

lm_eval/tasks/evalita_llm/sum_utils.py

+from evaluate import load
+
+
+rouge = load("rouge", keep_in_memory=True)
+
+
+def rouge1_score(references, predictions, **kwargs):
+    """
+    Optimized ROUGE-1 computation using a single loaded metric instance.
+    """
+    return rouge.compute(predictions=predictions, references=references, **kwargs)[
+        "rouge1"
+    ]
+
+
+def process_results_sum(doc, results):
+    """
+    Process the results of the summarization task efficiently.
+    """
+    ref = doc.get("summary", doc.get("target"))  # Get the reference summary
+    return {"rouge1": rouge.compute(predictions=results, references=[ref])["rouge1"]}

lm_eval/tasks/evalita_llm/utils.py

@@ -523,33 +523,6 @@ def split_text_with_regex(text, pattern):
     return result
 
 
-# ---------------------- SUMMARIZATION ----------------------
-
-
-def rouge1_score(references, predictions, **kwargs):
-    """
-    suboptimal way of compute rouge because of the following issue:
-    https://github.com/EleutherAI/lm-evaluation-harness/issues/1302
-    """
-    rouge = load("rouge")
-    return rouge.compute(predictions=predictions, references=references, **kwargs)[
-        "rouge1"
-    ]
-
-
-def process_results_sum(doc, results):
-    """
-    Process the results of the Evalita summarization task
-    """
-    ref = doc["summary"] if "summary" in doc.keys() else doc["target"]
-    rouge_scorer = load("rouge", keep_in_memory=True)
-    r1score = rouge_scorer.compute(predictions=results, references=[ref])["rouge1"]
-
-    return {
-        "rouge1": r1score,
-    }
-
-
 def faq_doc_to_target(x):
     if x["correct_answer"] == "A":
         return 0