metrics.py

import evaluate

rouge_fn = evaluate.load('rouge')

def rouge1(predictions, references):
    results = rouge_fn.compute(predictions=predictions, references=references)
    return results['rouge1']

def rouge2(predictions, references):
    results = rouge_fn.compute(predictions=predictions, references=references)
    return results['rouge2']

def rougeL(predictions, references):
    results = rouge_fn.compute(predictions=predictions, references=references)
    return results['rougeL']

squad_metric = evaluate.load("squad_v2")

def agg_f1(samples):
    predictions, references = zip(*samples)  # unzip, if you will
    computed = squad_metric.compute(predictions=predictions, references=references)
    return computed["f1"]


def _download_metric():
    import os
    import shutil
    from huggingface_hub import hf_hub_download
    scrolls_metric_path = hf_hub_download(repo_id="tau/scrolls", repo_type="dataset", filename="metrics/scrolls.py")
    updated_scrolls_metric_path = (
        os.path.dirname(scrolls_metric_path) + os.path.basename(scrolls_metric_path).replace(".", "_") + ".py"
    )
    shutil.copy(scrolls_metric_path, updated_scrolls_metric_path)
    return updated_scrolls_metric_path