base.py 7.58 KB
Newer Older
Leo Gao's avatar
Leo Gao committed
1
2
import abc
import random
thefazzer's avatar
thefazzer committed
3
import numpy as np
Jonathan Tow's avatar
Jonathan Tow committed
4
import sklearn
Jason Phang's avatar
gpt3  
Jason Phang committed
5

Jason Phang's avatar
Jason Phang committed
6

Leo Gao's avatar
Leo Gao committed
7
8
class LM(abc.ABC):
    @abc.abstractmethod
Leo Gao's avatar
Leo Gao committed
9
    def loglikelihood(self, requests):
Leo Gao's avatar
Leo Gao committed
10
11
12
        """Compute log-likelihood of generating a continuation from a context.
        Downstream tasks should attempt to use loglikelihood instead of other 
        LM calls whenever possible.
Jason Phang's avatar
gpt3  
Jason Phang committed
13

Leo Gao's avatar
Leo Gao committed
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
        :param requests: list
            A list of pairs (context, continuation)
            context: str
                Context string
            continuation: str
                The continuation over which log likelihood will be calculated. If 
                there is a word boundary, the space should be in the continuation. 
                For example, context="hello" continuation=" world" is correct.
        :return: list
            A list of pairs (logprob, isgreedy)
            logprob: float
                The log probability of `contination`
            isgreedy:
                Whether `contination` would be generated by greedy sampling from `context`
        """
        pass

    @abc.abstractmethod
Leo Gao's avatar
Update  
Leo Gao committed
32
    def greedy_until(self, requests):
Leo Gao's avatar
Leo Gao committed
33
34
35
36
37
38
39
40
        """Generate greedily until a stopping sequence

        :param requests: list
            A list of pairs (context, until)
            context: str
                Context string
            until: str
                The string sequence to generate until. This string sequence may 
Leo Gao's avatar
Leo Gao committed
41
                span across multiple tokens, or may be part of one token.
Leo Gao's avatar
Leo Gao committed
42
43
44
45
        :return: list
            A list of strings continuation
            continuation: str
                The generated continuation.
Jason Phang's avatar
gpt3  
Jason Phang committed
46
        """
Leo Gao's avatar
Leo Gao committed
47
48
        pass

Jason Phang's avatar
gpt3  
Jason Phang committed
49
50
51
52
53
54
55
56
57
58
59
    @classmethod
    def create_from_arg_string(cls, arg_string):
        """Constructor method, in case models need additional arguments
        e.g. OpenAI API engine, paths for loading, other params

        :param arg_string: str
            Left up to individual model class to handle

        """
        return cls()

Leo Gao's avatar
Leo Gao committed
60

61
class Task(abc.ABC):
Leo Gao's avatar
Leo Gao committed
62
63
    def __init__(self):
        self.download()
64
        self._training_docs = None
sdtblck's avatar
sdtblck committed
65
66
67
68
69

    def download(self):
        """Downloads the task dataset if necessary"""
        pass

70
71
    @abc.abstractmethod
    def has_training_docs(self):
Jason Phang's avatar
checkin  
Jason Phang committed
72
        """Whether the task has a training set"""
73
        pass
74

75
76
    @abc.abstractmethod
    def has_validation_docs(self):
Jason Phang's avatar
checkin  
Jason Phang committed
77
78
79
80
81
82
        """Whether the task has a validation set"""
        pass

    @abc.abstractmethod
    def has_test_docs(self):
        """Whether the task has a test set"""
83
84
        pass

Leo Gao's avatar
Leo Gao committed
85
    def training_docs(self):
Jason Phang's avatar
checkin  
Jason Phang committed
86
87
88
89
        """
        :return: Iterable[obj]
            A iterable of any object, that doc_to_text can handle
        """
Leo Gao's avatar
Leo Gao committed
90
        return []
91

Leo Gao's avatar
Leo Gao committed
92
    def validation_docs(self):
93
94
95
96
        """
        :return: Iterable[obj]
            A iterable of any object, that doc_to_text can handle
        """
Leo Gao's avatar
Leo Gao committed
97
        return []
98

Leo Gao's avatar
Leo Gao committed
99
    def test_docs(self):
100
101
102
103
        """
        :return: Iterable[obj]
            A iterable of any object, that doc_to_text can handle
        """
Leo Gao's avatar
Leo Gao committed
104
        return []
Leo Gao's avatar
Leo Gao committed
105

106
    def fewshot_examples(self, k):
107
108
109
        if self._training_docs is None:
            self._training_docs = list(self.training_docs())
        return random.sample(self._training_docs, k)
Leo Gao's avatar
Leo Gao committed
110
111

    @abc.abstractmethod
Leo Gao's avatar
Update  
Leo Gao committed
112
113
114
115
116
    def doc_to_text(self, doc):
        pass

    @abc.abstractmethod
    def doc_to_target(self, doc):
Leo Gao's avatar
Leo Gao committed
117
        pass
Leo Gao's avatar
Leo Gao committed
118
119

    @abc.abstractmethod
120
    def construct_requests(self, doc, ctx):
Leo Gao's avatar
Leo Gao committed
121
122
123
        """ Uses RequestFactory to construct Requests and returns an iterable of 
        Requests which will be sent to the LM.

124
125
        :param doc:
            The document as returned from training_docs, validation_docs, or test_docs.
Leo Gao's avatar
Leo Gao committed
126
        :param ctx: str
127
128
129
            The context string, generated by fewshot_context. This includes the natural 
            language description, as well as the few shot examples, and the question
            part of the document for `doc`. 
Leo Gao's avatar
Leo Gao committed
130
        """
Leo Gao's avatar
Leo Gao committed
131
        pass
132

Leo Gao's avatar
Leo Gao committed
133
    @abc.abstractmethod
Leo Gao's avatar
Leo Gao committed
134
    def process_results(self, doc, results):
Leo Gao's avatar
Update  
Leo Gao committed
135
        """Take a single document and the LM results and evaluates, returning a 
136
137
        dict where keys are the names of submetrics and values are the values of 
        the metric for that one document
Leo Gao's avatar
Leo Gao committed
138
139
140
141
142

        :param doc:
            The document as returned from training_docs, validation_docs, or test_docs.
        :param results:
            The results of the requests created in construct_requests.
Jason Phang's avatar
checkin  
Jason Phang committed
143
        """
Leo Gao's avatar
Leo Gao committed
144
        pass
Jason Phang's avatar
gpt3  
Jason Phang committed
145

146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
    @abc.abstractmethod
    def aggregation(self):
        """
        :returns: {str: [float] -> float}
            A dictionary where keys are the names of submetrics and values are 
            functions that aggregate a list of metrics
        """
        pass

    @abc.abstractmethod
    def higher_is_better(self):
        """
        :returns: {str: bool}
            A dictionary where keys are the names of submetrics and values are 
            whether a higher value of the submetric is better
        """
        pass

Jason Phang's avatar
Jason Phang committed
164
    def fewshot_description(self):
Jason Phang's avatar
checkin  
Jason Phang committed
165
166
        return ""

Jason Phang's avatar
Jason Phang committed
167
    def fewshot_context(self, doc, num_fewshot, provide_description):
Jason Phang's avatar
Jason Phang committed
168
        raw_description = self.fewshot_description()
Jason Phang's avatar
Jason Phang committed
169
        description = (raw_description + "\n===\n\n") if provide_description and raw_description else ""
170

171
172
173
174
175
176
        if num_fewshot == 0:
            labeled_examples = ""
        else:
            labeled_examples = "\n\n".join(
                [self.doc_to_text(doc) + self.doc_to_target(doc) for doc in self.fewshot_examples(k=num_fewshot)]
            ) + "\n\n"
Leo Gao's avatar
Update  
Leo Gao committed
177
178

        example = self.doc_to_text(doc).strip()
Leo Gao's avatar
Leo Gao committed
179
180
181
182
183
184
        return description + labeled_examples + example


def mean(arr):
    return sum(arr) / len(arr)

Jason Phang's avatar
Jason Phang committed
185

Jonathan Tow's avatar
Jonathan Tow committed
186
187
188
def median(arr):
    return arr[len(arr) // 2]

Jason Phang's avatar
Jason Phang committed
189

Jonathan Tow's avatar
Jonathan Tow committed
190
191
192
193
194
195
def matthews_corrcoef(items):
    unzipped_list = list(zip(*items))
    golds = unzipped_list[0]
    preds = unzipped_list[1]
    return sklearn.metrics.matthews_corrcoef(golds, preds)

Jason Phang's avatar
Jason Phang committed
196

thefazzer's avatar
thefazzer committed
197
198
199
200
def f1_score(items):
    unzipped_list = list(zip(*items))
    golds = unzipped_list[0]
    preds = unzipped_list[1]
Jonathan Tow's avatar
Jonathan Tow committed
201
    fscore = sklearn.metrics.f1_score(golds, preds)
thefazzer's avatar
thefazzer committed
202
203
    return max(fscore)

Jason Phang's avatar
Jason Phang committed
204

thefazzer's avatar
thefazzer committed
205
206
207
208
209
def acc_all(items):
    # Only count as correct if all answers are labeled correctly for each question
    question_scoring_dict = {}
    preds = list(zip(*items))[0]
    docs = list(zip(*items))[1]
Jason Phang's avatar
Jason Phang committed
210
211

    for doc, pred in zip(docs, preds):
thefazzer's avatar
thefazzer committed
212
213
214
        question_id = doc["idx"]["question"]
        if question_id not in question_scoring_dict:
            question_scoring_dict[question_id] = []
215
216
217

        gold_label = doc["label"] == 1
        question_scoring_dict[question_id].append(gold_label == pred)
thefazzer's avatar
thefazzer committed
218
219
220
221
            
    acc = np.mean([int(all(x)) for x in question_scoring_dict.values()])
    return acc

Jason Phang's avatar
Jason Phang committed
222
223
224
225
226
227
228
229
230
231

def metric_max_over_ground_truths(metric_fn, prediction, ground_truths):
    """Compute max metric between prediction and each ground truth."""
    scores_for_ground_truths = []
    for ground_truth in ground_truths:
        score = metric_fn(prediction, ground_truth)
        scores_for_ground_truths.append(score)
    return max(scores_for_ground_truths)


232
233
234
235
req_ret_lens = {
    'loglikelihood': 2
}

Jason Phang's avatar
Jason Phang committed
236

237
238
239
240
class Request:
    def __init__(self, type, args, index=None):
        if type not in req_ret_lens.keys():
            raise NotImplementedError('The request type {} is not implemented!'.format(type))
Leo Gao's avatar
Leo Gao committed
241

242
243
244
245
246
247
248
249
250
251
252
        self.type = type
        self.args = args
        self.index = index
    
    def __iter__(self):
        i = 0
        for i in range(req_ret_lens[self.type]):
            yield Request(self.type, self.args, i)
    
    def __getitem__(self, i):
        return Request(self.type, self.args, i)
Leo Gao's avatar
Leo Gao committed
253

Jason Phang's avatar
Jason Phang committed
254

Leo Gao's avatar
Leo Gao committed
255
256
class RequestFactory:
    def __getattr__(self, attr):
Leo Gao's avatar
Update  
Leo Gao committed
257
258
        def fn(*args):
            return Request(attr, args)
Leo Gao's avatar
Leo Gao committed
259
260
261
262
        return fn


rf = RequestFactory()