utils_multiple_choice.py 15.8 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
# coding=utf-8
# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Julien Chaumond's avatar
Julien Chaumond committed
16
""" Multiple choice fine-tuning: utilities to work with multiple choice tasks of reading comprehension """
17
18


Aymeric Augustin's avatar
Aymeric Augustin committed
19
20
21
import csv
import glob
import json
22
23
import logging
import os
Julien Chaumond's avatar
Julien Chaumond committed
24
25
26
from dataclasses import dataclass
from enum import Enum
from typing import List, Optional
Aymeric Augustin's avatar
Aymeric Augustin committed
27

Julien Chaumond's avatar
Julien Chaumond committed
28
import torch
Aymeric Augustin's avatar
Aymeric Augustin committed
29
import tqdm
Julien Chaumond's avatar
Julien Chaumond committed
30
from torch.utils.data.dataset import Dataset
Aymeric Augustin's avatar
Aymeric Augustin committed
31

Julien Chaumond's avatar
Julien Chaumond committed
32
from transformers import PreTrainedTokenizer, torch_distributed_zero_first
33
34
35
36
37


logger = logging.getLogger(__name__)


Julien Chaumond's avatar
Julien Chaumond committed
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
@dataclass(frozen=True)
class InputExample:
    """
    A single training/test example for multiple choice

    Args:
        example_id: Unique id for the example.
        question: string. The untokenized text of the second sequence (question).
        contexts: list of str. The untokenized text of the first sequence (context of corresponding question).
        endings: list of str. multiple choice's options. Its length must be equal to contexts' length.
        label: (Optional) string. The label of the example. This should be
        specified for train and dev examples, but not for test examples.
    """

    example_id: str
    question: str
    contexts: List[str]
    endings: List[str]
    label: Optional[str]


@dataclass(frozen=True)
class InputFeatures:
    """
    A single set of features of data.
    Property names are the same names as the corresponding inputs to a model.
    """

    example_id: str
    input_ids: List[List[int]]
    attention_mask: Optional[List[List[int]]]
    token_type_ids: Optional[List[List[int]]]
    label: Optional[int]


class Split(Enum):
    train = "train"
    dev = "dev"
    test = "test"


class MultipleChoiceDataset(Dataset):
    """
    This will be superseded by a framework-agnostic approach
    soon.
    """
84

Julien Chaumond's avatar
Julien Chaumond committed
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
    features: List[InputFeatures]

    def __init__(
        self,
        data_dir: str,
        tokenizer: PreTrainedTokenizer,
        task: str,
        max_seq_length: Optional[int] = None,
        overwrite_cache=False,
        mode: Split = Split.train,
        local_rank=-1,
    ):
        processor = processors[task]()

        cached_features_file = os.path.join(
            data_dir,
            "cached_{}_{}_{}_{}".format(mode.value, tokenizer.__class__.__name__, str(max_seq_length), task,),
        )
        with torch_distributed_zero_first(local_rank):
            # Make sure only the first process in distributed training processes the dataset,
            # and the others will use the cache.

            if os.path.exists(cached_features_file) and not overwrite_cache:
                logger.info(f"Loading features from cached file {cached_features_file}")
                self.features = torch.load(cached_features_file)
            else:
                logger.info(f"Creating features from dataset file at {data_dir}")
                label_list = processor.get_labels()
                if mode == Split.dev:
                    examples = processor.get_dev_examples(data_dir)
                elif mode == Split.test:
                    examples = processor.get_test_examples(data_dir)
                else:
                    examples = processor.get_train_examples(data_dir)
                logger.info("Training examples: %s", len(examples))
                # TODO clean up all this to leverage built-in features of tokenizers
                self.features = convert_examples_to_features(
                    examples,
                    label_list,
                    max_seq_length,
                    tokenizer,
                    pad_on_left=bool(tokenizer.padding_side == "left"),
                    pad_token=tokenizer.pad_token_id,
                    pad_token_segment_id=tokenizer.pad_token_type_id,
                )
                if local_rank in [-1, 0]:
                    logger.info("Saving features into cached file %s", cached_features_file)
                    torch.save(self.features, cached_features_file)
133

Julien Chaumond's avatar
Julien Chaumond committed
134
135
136
137
138
139
140
141
    def __len__(self):
        return len(self.features)

    def __getitem__(self, i) -> InputFeatures:
        return self.features[i]


class DataProcessor:
erenup's avatar
erenup committed
142
    """Base class for data converters for multiple choice data sets."""
143
144
145
146
147
148
149
150
151

    def get_train_examples(self, data_dir):
        """Gets a collection of `InputExample`s for the train set."""
        raise NotImplementedError()

    def get_dev_examples(self, data_dir):
        """Gets a collection of `InputExample`s for the dev set."""
        raise NotImplementedError()

erenup's avatar
erenup committed
152
    def get_test_examples(self, data_dir):
erenup's avatar
erenup committed
153
        """Gets a collection of `InputExample`s for the test set."""
erenup's avatar
erenup committed
154
155
        raise NotImplementedError()

156
157
158
159
160
161
    def get_labels(self):
        """Gets the list of labels for this data set."""
        raise NotImplementedError()


class RaceProcessor(DataProcessor):
erenup's avatar
erenup committed
162
    """Processor for the RACE data set."""
163
164
165
166

    def get_train_examples(self, data_dir):
        """See base class."""
        logger.info("LOOKING AT {} train".format(data_dir))
167
168
        high = os.path.join(data_dir, "train/high")
        middle = os.path.join(data_dir, "train/middle")
169
170
        high = self._read_txt(high)
        middle = self._read_txt(middle)
171
        return self._create_examples(high + middle, "train")
172
173
174
175

    def get_dev_examples(self, data_dir):
        """See base class."""
        logger.info("LOOKING AT {} dev".format(data_dir))
176
177
        high = os.path.join(data_dir, "dev/high")
        middle = os.path.join(data_dir, "dev/middle")
178
179
        high = self._read_txt(high)
        middle = self._read_txt(middle)
180
        return self._create_examples(high + middle, "dev")
181

erenup's avatar
erenup committed
182
183
184
    def get_test_examples(self, data_dir):
        """See base class."""
        logger.info("LOOKING AT {} test".format(data_dir))
185
186
        high = os.path.join(data_dir, "test/high")
        middle = os.path.join(data_dir, "test/middle")
erenup's avatar
erenup committed
187
188
        high = self._read_txt(high)
        middle = self._read_txt(middle)
189
        return self._create_examples(high + middle, "test")
erenup's avatar
erenup committed
190

191
192
193
194
195
196
197
198
    def get_labels(self):
        """See base class."""
        return ["0", "1", "2", "3"]

    def _read_txt(self, input_dir):
        lines = []
        files = glob.glob(input_dir + "/*txt")
        for file in tqdm.tqdm(files, desc="read files"):
199
            with open(file, "r", encoding="utf-8") as fin:
200
201
202
203
204
205
206
207
208
209
210
211
                data_raw = json.load(fin)
                data_raw["race_id"] = file
                lines.append(data_raw)
        return lines

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for (_, data_raw) in enumerate(lines):
            race_id = "%s-%s" % (set_type, data_raw["race_id"])
            article = data_raw["article"]
            for i in range(len(data_raw["answers"])):
212
213
214
                truth = str(ord(data_raw["answers"][i]) - ord("A"))
                question = data_raw["questions"][i]
                options = data_raw["options"][i]
215
216
217
218
219

                examples.append(
                    InputExample(
                        example_id=race_id,
                        question=question,
220
                        contexts=[article, article, article, article],  # this is not efficient but convenient
221
                        endings=[options[0], options[1], options[2], options[3]],
222
223
224
                        label=truth,
                    )
                )
225
226
        return examples

227

228
class SwagProcessor(DataProcessor):
erenup's avatar
erenup committed
229
    """Processor for the SWAG data set."""
230
231
232
233
234
235
236
237
238
239
240

    def get_train_examples(self, data_dir):
        """See base class."""
        logger.info("LOOKING AT {} train".format(data_dir))
        return self._create_examples(self._read_csv(os.path.join(data_dir, "train.csv")), "train")

    def get_dev_examples(self, data_dir):
        """See base class."""
        logger.info("LOOKING AT {} dev".format(data_dir))
        return self._create_examples(self._read_csv(os.path.join(data_dir, "val.csv")), "dev")

erenup's avatar
erenup committed
241
242
    def get_test_examples(self, data_dir):
        """See base class."""
erenup's avatar
erenup committed
243
244
245
246
247
        logger.info("LOOKING AT {} dev".format(data_dir))
        raise ValueError(
            "For swag testing, the input file does not contain a label column. It can not be tested in current code"
            "setting!"
        )
erenup's avatar
erenup committed
248
        return self._create_examples(self._read_csv(os.path.join(data_dir, "test.csv")), "test")
249

250
251
252
253
254
    def get_labels(self):
        """See base class."""
        return ["0", "1", "2", "3"]

    def _read_csv(self, input_file):
255
        with open(input_file, "r", encoding="utf-8") as f:
256
            return list(csv.reader(f))
257

258
    def _create_examples(self, lines: List[List[str]], type: str):
259
        """Creates examples for the training and dev sets."""
260
261
        if type == "train" and lines[0][-1] != "label":
            raise ValueError("For training, the input file must contain a label column.")
262
263
264
265
266
267
268

        examples = [
            InputExample(
                example_id=line[2],
                question=line[5],  # in the swag dataset, the
                # common beginning of each
                # choice is stored in "sent2".
269
270
271
272
273
                contexts=[line[4], line[4], line[4], line[4]],
                endings=[line[7], line[8], line[9], line[10]],
                label=line[11],
            )
            for line in lines[1:]  # we skip the line with the column names
274
275
276
277
278
279
        ]

        return examples


class ArcProcessor(DataProcessor):
erenup's avatar
erenup committed
280
    """Processor for the ARC data set (request from allennlp)."""
281
282
283
284
285
286
287
288
289
290
291

    def get_train_examples(self, data_dir):
        """See base class."""
        logger.info("LOOKING AT {} train".format(data_dir))
        return self._create_examples(self._read_json(os.path.join(data_dir, "train.jsonl")), "train")

    def get_dev_examples(self, data_dir):
        """See base class."""
        logger.info("LOOKING AT {} dev".format(data_dir))
        return self._create_examples(self._read_json(os.path.join(data_dir, "dev.jsonl")), "dev")

erenup's avatar
erenup committed
292
293
294
295
    def get_test_examples(self, data_dir):
        logger.info("LOOKING AT {} test".format(data_dir))
        return self._create_examples(self._read_json(os.path.join(data_dir, "test.jsonl")), "test")

296
297
298
299
300
    def get_labels(self):
        """See base class."""
        return ["0", "1", "2", "3"]

    def _read_json(self, input_file):
301
        with open(input_file, "r", encoding="utf-8") as fin:
302
303
304
305
306
307
            lines = fin.readlines()
            return lines

    def _create_examples(self, lines, type):
        """Creates examples for the training and dev sets."""

308
        # There are two types of labels. They should be normalized
309
310
311
312
313
314
        def normalize(truth):
            if truth in "ABCD":
                return ord(truth) - ord("A")
            elif truth in "1234":
                return int(truth) - 1
            else:
erenup's avatar
erenup committed
315
316
                logger.info("truth ERROR! %s", str(truth))
                return None
erenup's avatar
erenup committed
317

318
319
320
321
322
        examples = []
        three_choice = 0
        four_choice = 0
        five_choice = 0
        other_choices = 0
erenup's avatar
erenup committed
323
        # we deleted example which has more than or less than four choices
324
325
326
327
328
329
330
331
332
333
334
335
336
        for line in tqdm.tqdm(lines, desc="read arc data"):
            data_raw = json.loads(line.strip("\n"))
            if len(data_raw["question"]["choices"]) == 3:
                three_choice += 1
                continue
            elif len(data_raw["question"]["choices"]) == 5:
                five_choice += 1
                continue
            elif len(data_raw["question"]["choices"]) != 4:
                other_choices += 1
                continue
            four_choice += 1
            truth = str(normalize(data_raw["answerKey"]))
erenup's avatar
erenup committed
337
            assert truth != "None"
338
339
340
341
342
343
344
            question_choices = data_raw["question"]
            question = question_choices["stem"]
            id = data_raw["id"]
            options = question_choices["choices"]
            if len(options) == 4:
                examples.append(
                    InputExample(
345
                        example_id=id,
346
                        question=question,
347
348
349
350
351
352
                        contexts=[
                            options[0]["para"].replace("_", ""),
                            options[1]["para"].replace("_", ""),
                            options[2]["para"].replace("_", ""),
                            options[3]["para"].replace("_", ""),
                        ],
353
                        endings=[options[0]["text"], options[1]["text"], options[2]["text"], options[3]["text"]],
354
355
356
                        label=truth,
                    )
                )
357
358
359
360
361
362
363
364
365
366
367
368
369

        if type == "train":
            assert len(examples) > 1
            assert examples[0].label is not None
        logger.info("len examples: %s}", str(len(examples)))
        logger.info("Three choices: %s", str(three_choice))
        logger.info("Five choices: %s", str(five_choice))
        logger.info("Other choices: %s", str(other_choices))
        logger.info("four choices: %s", str(four_choice))

        return examples


370
371
372
373
374
375
376
377
378
379
380
381
def convert_examples_to_features(
    examples: List[InputExample],
    label_list: List[str],
    max_length: int,
    tokenizer: PreTrainedTokenizer,
    pad_token_segment_id=0,
    pad_on_left=False,
    pad_token=0,
    mask_padding_with_zero=True,
) -> List[InputFeatures]:
    """
    Loads a data file into a list of `InputFeatures`
382
383
    """

384
    label_map = {label: i for i, label in enumerate(label_list)}
385
386
387
388
389

    features = []
    for (ex_index, example) in tqdm.tqdm(enumerate(examples), desc="convert examples to features"):
        if ex_index % 10000 == 0:
            logger.info("Writing example %d of %d" % (ex_index, len(examples)))
Julien Chaumond's avatar
Julien Chaumond committed
390
        choices_inputs = []
391
        for ending_idx, (context, ending) in enumerate(zip(example.contexts, example.endings)):
392
            text_a = context
393
            if example.question.find("_") != -1:
394
395
                # this is for cloze question
                text_b = example.question.replace("_", ending)
396
            else:
397
398
                text_b = example.question + " " + ending

399
            inputs = tokenizer.encode_plus(
Julien Chaumond's avatar
Julien Chaumond committed
400
                text_a, text_b, add_special_tokens=True, max_length=max_length, pad_to_max_length=True,
401
            )
402
403
404
405
406
407
            if "num_truncated_tokens" in inputs and inputs["num_truncated_tokens"] > 0:
                logger.info(
                    "Attention! you are cropping tokens (swag task is ok). "
                    "If you are training ARC and RACE and you are poping question + options,"
                    "you need to try to use a bigger max seq length!"
                )
408

Julien Chaumond's avatar
Julien Chaumond committed
409
            choices_inputs.append(inputs)
410

Julien Chaumond's avatar
Julien Chaumond committed
411
        label = label_map[example.label]
412

Julien Chaumond's avatar
Julien Chaumond committed
413
414
415
416
417
418
419
        input_ids = [x["input_ids"] for x in choices_inputs]
        attention_mask = (
            [x["attention_mask"] for x in choices_inputs] if "attention_mask" in choices_inputs[0] else None
        )
        token_type_ids = (
            [x["token_type_ids"] for x in choices_inputs] if "token_type_ids" in choices_inputs[0] else None
        )
420

Julien Chaumond's avatar
Julien Chaumond committed
421
422
423
424
425
426
427
428
429
        features.append(
            InputFeatures(
                example_id=example.example_id,
                input_ids=input_ids,
                attention_mask=attention_mask,
                token_type_ids=token_type_ids,
                label=label,
            )
        )
430

Julien Chaumond's avatar
Julien Chaumond committed
431
432
433
    for f in features[:2]:
        logger.info("*** Example ***")
        logger.info("feature: %s" % f)
434
435
436
437

    return features


438
processors = {"race": RaceProcessor, "swag": SwagProcessor, "arc": ArcProcessor}
439
440


441
MULTIPLE_CHOICE_TASKS_NUM_LABELS = {"race", 4, "swag", 4, "arc", 4}