t5_utils.py

import collections

import numpy as np
import sklearn.metrics

def f1(predictions, references):  # This is a passthrough function

    _prediction = predictions[0]
    _reference = references[0].split("_")[-1]
    string_label = ['False', 'True']
    reference = string_label.index(_reference)
    prediction = string_label.index(_prediction) if _prediction in string_label else not bool(reference)

    return (prediction, reference)

def agg_f1(items):

    predictions, references = zip(*items)
    references, predictions = np.asarray(references), np.asarray(predictions)

    return sklearn.metrics.f1_score(references, predictions)


def em(predictions, references):  # This is a passthrough function

    _prediction = predictions[0]
    _group, _reference = references[0].split("_")
    string_label = ['False', 'True']
    reference = string_label.index(_reference)
    prediction = string_label.index(_prediction) if _prediction in string_label else not bool(reference)

    return (_group, prediction, reference)


def agg_em(items):

    grouped_values = collections.defaultdict(lambda: ([], []))
    for group, prediction, reference in items:
        grouped_values[group][0].append(reference)
        grouped_values[group][1].append(prediction)

    group_scores = []
    for group, (targets, predictions) in grouped_values.items():
        score = float(np.array_equal(targets, predictions))
        group_scores.append(score)

    return np.mean(group_scores)