mutual.py 3.07 KB
Newer Older
Jonathan Tow's avatar
Jonathan Tow committed
1
2
3
4
"""
MuTual: A Dataset for Multi-Turn Dialogue Reasoning
https://www.aclweb.org/anthology/2020.acl-main.130/

5
6
7
8
MuTual is a retrieval-based dataset for multi-turn dialogue reasoning, which is
modified from Chinese high school English listening comprehension test data.

Homepage: https://github.com/Nealcly/MuTual
Jonathan Tow's avatar
Jonathan Tow committed
9
10
"""
import numpy as np
Jonathan Tow's avatar
Jonathan Tow committed
11
12
import inspect
import lm_eval.datasets.mutual.mutual
Jonathan Tow's avatar
Jonathan Tow committed
13
14
from lm_eval.base import Task, rf
from lm_eval.metrics import mean
Jonathan Tow's avatar
Jonathan Tow committed
15

Jonathan Tow's avatar
Jonathan Tow committed
16

17
18
19
20
21
22
23
24
25
26
27
_CITATION = """
@inproceedings{mutual,
    title = "MuTual: A Dataset for Multi-Turn Dialogue Reasoning",
    author = "Cui, Leyang  and Wu, Yu and Liu, Shujie and Zhang, Yue and Zhou, Ming" ,
    booktitle = "Proceedings of the 58th Conference of the Association for Computational Linguistics",
    year = "2020",
    publisher = "Association for Computational Linguistics",
}
"""


Jonathan Tow's avatar
Jonathan Tow committed
28
class MuTualBase(Task):
29
    VERSION = 1
Jonathan Tow's avatar
Jonathan Tow committed
30
    DATASET_PATH = inspect.getfile(lm_eval.datasets.mutual.mutual)
Jonathan Tow's avatar
Jonathan Tow committed
31
    DATASET_NAME = None
bzantium's avatar
bzantium committed
32
    CHOICES = ["A", "B", "C", "D"]
Jonathan Tow's avatar
Jonathan Tow committed
33
34
35
36
37
38
39
40
41
42
43

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
        return False

    def training_docs(self):
Jonathan Tow's avatar
Jonathan Tow committed
44
        return self.dataset["train"]
Jonathan Tow's avatar
Jonathan Tow committed
45
46

    def validation_docs(self):
Jonathan Tow's avatar
Jonathan Tow committed
47
        return self.dataset["validation"]
Jonathan Tow's avatar
Jonathan Tow committed
48
49
50
51
52

    def test_docs(self):
        return NotImplemented

    def doc_to_text(self, doc):
Jonathan Tow's avatar
Jonathan Tow committed
53
        return self.detokenize(doc["article"])
Jonathan Tow's avatar
Jonathan Tow committed
54

bzantium's avatar
bzantium committed
55
56
57
58
59
60
    def should_decontaminate(self):
        return True

    def doc_to_decontamination_query(self, doc):
        return doc["article"]

Jonathan Tow's avatar
Jonathan Tow committed
61
    def doc_to_target(self, doc):
Jonathan Tow's avatar
Jonathan Tow committed
62
        return " " + self.detokenize(doc["options"][self.CHOICES.index(doc["answers"])])
Jonathan Tow's avatar
Jonathan Tow committed
63
64
65
66

    def construct_requests(self, doc, ctx):
        lls = []
        for option in doc["options"]:
67
            lls.append(rf.loglikelihood(ctx, f" {self.detokenize(option)}")[0])
Jonathan Tow's avatar
Jonathan Tow committed
68
69
        return lls

Jonathan Tow's avatar
Jonathan Tow committed
70
71
72
73
74
75
76
77
78
79
80
81
82
83
    def detokenize(self, text):
        text = text.replace(" '", "'")
        text = text.replace(" \n", "\n")
        text = text.replace("\n ", "\n")
        text = text.replace(" n't", "n't")
        text = text.replace("`` ", '"')
        text = text.replace("''", '"')
        # punctuation
        text = text.replace(" :", ":")
        text = text.replace(" ;", ";")
        text = text.replace(" !", "!")
        text = text.replace(" ?", "?")
        text = text.replace(" ,", ",")
        text = text.replace(" .", ".")
Leo Gao's avatar
Leo Gao committed
84
        return text
Jonathan Tow's avatar
Jonathan Tow committed
85

Jonathan Tow's avatar
Jonathan Tow committed
86
87
88
89
90
    def process_results(self, doc, results):
        gold = self.CHOICES.index(doc["answers"])
        r4_1 = np.argmax(results) == gold  # r4_1 = accuracy
        ranks = sorted(results, reverse=True)
        r4_2 = (ranks.index(results[gold]) == 1) + r4_1
bzantium's avatar
bzantium committed
91
92
        mrr = 1.0 / (ranks.index(results[gold]) + 1)  # `+ 1` for index offset
        return {"r@1": r4_1, "r@2": r4_2, "mrr": mrr}
Jonathan Tow's avatar
Jonathan Tow committed
93
94

    def aggregation(self):
bzantium's avatar
bzantium committed
95
        return {"r@1": mean, "r@2": mean, "mrr": mean}
Jonathan Tow's avatar
Jonathan Tow committed
96
97

    def higher_is_better(self):
bzantium's avatar
bzantium committed
98
        return {"r@1": True, "r@2": True, "mrr": True}
Jonathan Tow's avatar
Jonathan Tow committed
99
100
101


class MuTual(MuTualBase):
Jonathan Tow's avatar
Jonathan Tow committed
102
    DATASET_NAME = "mutual"
Jonathan Tow's avatar
Jonathan Tow committed
103
104
105


class MuTualPlus(MuTualBase):
Jonathan Tow's avatar
Jonathan Tow committed
106
    DATASET_NAME = "mutual_plus"