utils.py

import datasets
import evaluate


def strip(resps, docs):
    """
    Assuming each entry of `resps` is a list of model responses, we discard all but the first response.
    """
    return map(lambda r: r[0].strip(), resps)


def dr_fr(dataset: datasets.Dataset):
    return dataset.filter(lambda x: x["direction"] == "dr_fr")


def dr_en(dataset: datasets.Dataset):
    return dataset.filter(lambda x: x["direction"] == "dr_en")


def dr_msa(dataset: datasets.Dataset):
    return dataset.filter(lambda x: x["direction"] == "dr_msa")


def fr_dr(dataset: datasets.Dataset):
    return dataset.filter(lambda x: x["direction"] == "fr_dr")


def en_dr(dataset: datasets.Dataset):
    return dataset.filter(lambda x: x["direction"] == "en_dr")


def msa_dr(dataset: datasets.Dataset):
    return dataset.filter(lambda x: x["direction"] == "msa_dr")


prompt_templates = {
    "fr_dr": "ترجم من الفرنساوية للدارجة:\n{0}",
    "dr_fr": "ترجم من الدارجة للفرنساوية:\n{0}",
    "en_dr": "ترجم من الإنجليزية للدارجة:\n{0}",
    "dr_en": "ترجم من الدارجة للإنجليزية:\n{0}",
    "msa_dr": "ترجم من الفصحى للدارجة:\n{0}",
    "dr_msa": "ترجم من الدارجة للفصحى:\n{0}",
}


def doc_to_text(doc):
    doc_text = doc["messages"][0]["content"]
    return doc_text


def doc_to_target(doc):
    return doc["messages"][1]["content"]


def bert(items):
    return items


def Average(lst):
    return sum(lst) / len(lst)


def camembert(items):
    bert_model = "almanach/camembert-base"
    bert_score = evaluate.load("bertscore")
    predictions, references = zip(*items)
    bert = bert_score.compute(
        predictions=predictions,
        references=references,
        model_type=bert_model,
        num_layers=12,
    )
    return Average(bert["f1"])


def darijabert(items):
    bert_model = "SI2M-Lab/DarijaBERT"
    bert_score = evaluate.load("bertscore")
    predictions, references = zip(*items)
    bert = bert_score.compute(
        predictions=predictions,
        references=references,
        model_type=bert_model,
        num_layers=12,
    )
    return Average(bert["f1"])


def arabert(items):
    bert_model = "aubmindlab/bert-base-arabert"
    bert_score = evaluate.load("bertscore")
    predictions, references = zip(*items)
    bert = bert_score.compute(
        predictions=predictions,
        references=references,
        model_type=bert_model,
        num_layers=12,
    )
    return Average(bert["f1"])


def bertbase(items):
    bert_model = "google-bert/bert-base-uncased"
    bert_score = evaluate.load("bertscore")
    predictions, references = zip(*items)
    bert = bert_score.compute(
        predictions=predictions,
        references=references,
        model_type=bert_model,
        num_layers=12,
    )
    return Average(bert["f1"])


def mbert(items):
    bert_model = "google-bert/bert-base-multilingual-cased"
    bert_score = evaluate.load("bertscore")
    predictions, references = zip(*items)
    bert = bert_score.compute(
        predictions=predictions,
        references=references,
        model_type=bert_model,
        num_layers=12,
    )
    return Average(bert["f1"])