run_glue.py 17.5 KB
Newer Older
thomwolf's avatar
thomwolf committed
1
# coding=utf-8
Sylvain Gugger's avatar
Sylvain Gugger committed
2
# Copyright 2020 The HuggingFace Inc. team. All rights reserved.
thomwolf's avatar
thomwolf committed
3
4
5
6
7
8
9
10
11
12
13
14
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Lysandre's avatar
Lysandre committed
15
""" Finetuning the library models for sequence classification on GLUE."""
Sylvain Gugger's avatar
Sylvain Gugger committed
16
# You can also adapt this script on your own text classification task. Pointers for this are left as comments.
thomwolf's avatar
thomwolf committed
17
18
19

import logging
import os
Sylvain Gugger's avatar
Sylvain Gugger committed
20
import random
21
import sys
22
from dataclasses import dataclass, field
Sylvain Gugger's avatar
Sylvain Gugger committed
23
from typing import Optional
thomwolf's avatar
thomwolf committed
24
25

import numpy as np
Sylvain Gugger's avatar
Sylvain Gugger committed
26
from datasets import load_dataset, load_metric
thomwolf's avatar
thomwolf committed
27

Sylvain Gugger's avatar
Sylvain Gugger committed
28
import transformers
29
from transformers import (
Sylvain Gugger's avatar
Sylvain Gugger committed
30
31
32
33
    AutoConfig,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    EvalPrediction,
34
    HfArgumentParser,
Sylvain Gugger's avatar
Sylvain Gugger committed
35
    PretrainedConfig,
Julien Chaumond's avatar
Julien Chaumond committed
36
    Trainer,
37
    TrainingArguments,
Sylvain Gugger's avatar
Sylvain Gugger committed
38
    default_data_collator,
Julien Chaumond's avatar
Julien Chaumond committed
39
    set_seed,
40
)
Sylvain Gugger's avatar
Sylvain Gugger committed
41
42
from transformers.trainer_utils import is_main_process

Aymeric Augustin's avatar
Aymeric Augustin committed
43

Sylvain Gugger's avatar
Sylvain Gugger committed
44
45
46
47
48
49
50
51
52
53
54
task_to_keys = {
    "cola": ("sentence", None),
    "mnli": ("premise", "hypothesis"),
    "mrpc": ("sentence1", "sentence2"),
    "qnli": ("question", "sentence"),
    "qqp": ("question1", "question2"),
    "rte": ("sentence1", "sentence2"),
    "sst2": ("sentence", None),
    "stsb": ("sentence1", "sentence2"),
    "wnli": ("sentence1", "sentence2"),
}
thomwolf's avatar
thomwolf committed
55
56
57

logger = logging.getLogger(__name__)

thomwolf's avatar
thomwolf committed
58

Sylvain Gugger's avatar
Sylvain Gugger committed
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
@dataclass
class DataTrainingArguments:
    """
    Arguments pertaining to what data we are going to input our model for training and eval.

    Using `HfArgumentParser` we can turn this class
    into argparse arguments to be able to specify them on
    the command line.
    """

    task_name: Optional[str] = field(
        default=None,
        metadata={"help": "The name of the task to train on: " + ", ".join(task_to_keys.keys())},
    )
    max_seq_length: int = field(
        default=128,
        metadata={
            "help": "The maximum total input sequence length after tokenization. Sequences longer "
            "than this will be truncated, sequences shorter will be padded."
        },
    )
    overwrite_cache: bool = field(
        default=False, metadata={"help": "Overwrite the cached preprocessed datasets or not."}
    )
    pad_to_max_length: bool = field(
        default=True,
        metadata={
            "help": "Whether to pad all samples to `max_seq_length`. "
            "If False, will pad the samples dynamically when batching to the maximum length in the batch."
        },
    )
    train_file: Optional[str] = field(
        default=None, metadata={"help": "A csv or a json file containing the training data."}
    )
    validation_file: Optional[str] = field(
        default=None, metadata={"help": "A csv or a json file containing the validation data."}
    )

    def __post_init__(self):
        if self.task_name is not None:
            self.task_name = self.task_name.lower()
            if self.task_name not in task_to_keys.keys():
                raise ValueError("Unknown task, you should pick one in " + ",".join(task_to_keys.keys()))
        elif self.train_file is None or self.validation_file is None:
            raise ValueError("Need either a GLUE task or a training/validation file.")
        else:
            extension = self.train_file.split(".")[-1]
            assert extension in ["csv", "json"], "`train_file` should be a csv or a json file."
            extension = self.validation_file.split(".")[-1]
            assert extension in ["csv", "json"], "`validation_file` should be a csv or a json file."


111
112
113
114
115
116
117
@dataclass
class ModelArguments:
    """
    Arguments pertaining to which model/config/tokenizer we are going to fine-tune from.
    """

    model_name_or_path: str = field(
Julien Chaumond's avatar
Julien Chaumond committed
118
        metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}
119
    )
120
121
    config_name: Optional[str] = field(
        default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
122
    )
123
124
    tokenizer_name: Optional[str] = field(
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
125
    )
126
    cache_dir: Optional[str] = field(
Julien Chaumond's avatar
Julien Chaumond committed
127
        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
128
    )
Sylvain Gugger's avatar
Sylvain Gugger committed
129
130
131
132
    use_fast_tokenizer: bool = field(
        default=True,
        metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."},
    )
133
134


135
def main():
Julien Chaumond's avatar
Julien Chaumond committed
136
137
138
    # See all possible arguments in src/transformers/training_args.py
    # or by passing the --help flag to this script.
    # We now keep distinct sets of args, for a cleaner separation of concerns.
139

140
141
142
143
144
145
146
    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
    if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
        # If we pass only one argument to the script and it's the path to a json file,
        # let's parse it to get our arguments.
        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
    else:
        model_args, data_args, training_args = parser.parse_args_into_dataclasses()
thomwolf's avatar
thomwolf committed
147

148
    if (
Julien Chaumond's avatar
Julien Chaumond committed
149
150
151
152
        os.path.exists(training_args.output_dir)
        and os.listdir(training_args.output_dir)
        and training_args.do_train
        and not training_args.overwrite_output_dir
153
154
    ):
        raise ValueError(
Sylvain Gugger's avatar
Sylvain Gugger committed
155
156
            f"Output directory ({training_args.output_dir}) already exists and is not empty. "
            "Use --overwrite_output_dir to overcome."
157
        )
thomwolf's avatar
thomwolf committed
158

thomwolf's avatar
thomwolf committed
159
    # Setup logging
160
161
162
    logging.basicConfig(
        format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
        datefmt="%m/%d/%Y %H:%M:%S",
Sylvain Gugger's avatar
Sylvain Gugger committed
163
        level=logging.INFO if is_main_process(training_args.local_rank) else logging.WARN,
164
    )
Sylvain Gugger's avatar
Sylvain Gugger committed
165
166

    # Log on each process the small summary:
167
    logger.warning(
Sylvain Gugger's avatar
Sylvain Gugger committed
168
169
        f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"
        + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}"
170
    )
Sylvain Gugger's avatar
Sylvain Gugger committed
171
172
173
    # Set the verbosity to info of the Transformers logger (on main process only):
    if is_main_process(training_args.local_rank):
        transformers.utils.logging.set_verbosity_info()
174
175
        transformers.utils.logging.enable_default_handler()
        transformers.utils.logging.enable_explicit_format()
Sylvain Gugger's avatar
Sylvain Gugger committed
176
    logger.info(f"Training/evaluation parameters {training_args}")
thomwolf's avatar
thomwolf committed
177

Sylvain Gugger's avatar
Sylvain Gugger committed
178
    # Set seed before initializing model.
Julien Chaumond's avatar
Julien Chaumond committed
179
    set_seed(training_args.seed)
thomwolf's avatar
thomwolf committed
180

Sylvain Gugger's avatar
Sylvain Gugger committed
181
    # Get the datasets: you can either provide your own CSV/JSON training and evaluation files (see below)
Sylvain Gugger's avatar
Sylvain Gugger committed
182
    # or specify a GLUE benchmark task (the dataset will be downloaded automatically from the datasets Hub).
Sylvain Gugger's avatar
Sylvain Gugger committed
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
    #
    # For CSV/JSON files, this script will use as labels the column called 'label' and as pair of sentences the
    # sentences in columns called 'sentence1' and 'sentence2' if such column exists or the first two columns not named
    # label if at least two columns are provided.
    #
    # If the CSVs/JSONs contain only one non-label column, the script does single sentence classification on this
    # single column. You can easily tweak this behavior (see below)
    #
    # In distributed training, the load_dataset function guarantee that only one local process can concurrently
    # download the dataset.
    if data_args.task_name is not None:
        # Downloading and loading a dataset from the hub.
        datasets = load_dataset("glue", data_args.task_name)
    elif data_args.train_file.endswith(".csv"):
        # Loading a dataset from local csv files
        datasets = load_dataset(
            "csv", data_files={"train": data_args.train_file, "validation": data_args.validation_file}
        )
    else:
        # Loading a dataset from local json files
        datasets = load_dataset(
            "json", data_files={"train": data_args.train_file, "validation": data_args.validation_file}
        )
    # See more about loading any type of standard or custom dataset at
    # https://huggingface.co/docs/datasets/loading_datasets.html.

    # Labels
    if data_args.task_name is not None:
        is_regression = data_args.task_name == "stsb"
        if not is_regression:
            label_list = datasets["train"].features["label"].names
            num_labels = len(label_list)
        else:
            num_labels = 1
    else:
        # Trying to have good defaults here, don't hesitate to tweak to your needs.
        is_regression = datasets["train"].features["label"].dtype in ["float32", "float64"]
        if is_regression:
            num_labels = 1
        else:
            # A useful fast method:
            # https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.unique
            label_list = datasets["train"].unique("label")
            label_list.sort()  # Let's sort it for determinism
            num_labels = len(label_list)
thomwolf's avatar
thomwolf committed
228
229

    # Load pretrained model and tokenizer
Julien Chaumond's avatar
Julien Chaumond committed
230
    #
Sylvain Gugger's avatar
Sylvain Gugger committed
231
    # In distributed training, the .from_pretrained methods guarantee that only one local process can concurrently
Julien Chaumond's avatar
Julien Chaumond committed
232
    # download model & vocab.
233
    config = AutoConfig.from_pretrained(
Julien Chaumond's avatar
Julien Chaumond committed
234
        model_args.config_name if model_args.config_name else model_args.model_name_or_path,
235
        num_labels=num_labels,
Julien Chaumond's avatar
Julien Chaumond committed
236
237
        finetuning_task=data_args.task_name,
        cache_dir=model_args.cache_dir,
238
    )
239
    tokenizer = AutoTokenizer.from_pretrained(
Julien Chaumond's avatar
Julien Chaumond committed
240
241
        model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,
        cache_dir=model_args.cache_dir,
Sylvain Gugger's avatar
Sylvain Gugger committed
242
        use_fast=model_args.use_fast_tokenizer,
243
    )
244
    model = AutoModelForSequenceClassification.from_pretrained(
Julien Chaumond's avatar
Julien Chaumond committed
245
246
        model_args.model_name_or_path,
        from_tf=bool(".ckpt" in model_args.model_name_or_path),
247
        config=config,
Julien Chaumond's avatar
Julien Chaumond committed
248
        cache_dir=model_args.cache_dir,
249
    )
thomwolf's avatar
thomwolf committed
250

Sylvain Gugger's avatar
Sylvain Gugger committed
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
    # Preprocessing the datasets
    if data_args.task_name is not None:
        sentence1_key, sentence2_key = task_to_keys[data_args.task_name]
    else:
        # Again, we try to have some nice defaults but don't hesitate to tweak to your use case.
        non_label_column_names = [name for name in datasets["train"].column_names if name != "label"]
        if "sentence1" in non_label_column_names and "sentence2" in non_label_column_names:
            sentence1_key, sentence2_key = "sentence1", "sentence2"
        else:
            if len(non_label_column_names) >= 2:
                sentence1_key, sentence2_key = non_label_column_names[:2]
            else:
                sentence1_key, sentence2_key = non_label_column_names[0], None

    # Padding strategy
    if data_args.pad_to_max_length:
        padding = "max_length"
        max_length = data_args.max_seq_length
    else:
        # We will pad later, dynamically at batch creation, to the max sequence length in each batch
        padding = False
        max_length = None
thomwolf's avatar
thomwolf committed
273

Sylvain Gugger's avatar
Sylvain Gugger committed
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
    # Some models have set the order of the labels to use, so let's make sure we do use it.
    label_to_id = None
    if (
        model.config.label2id != PretrainedConfig(num_labels=num_labels).label2id
        and data_args.task_name is not None
        and is_regression
    ):
        # Some have all caps in their config, some don't.
        label_name_to_id = {k.lower(): v for k, v in model.config.label2id.items()}
        if list(sorted(label_name_to_id.keys())) == list(sorted(label_list)):
            label_to_id = {i: label_name_to_id[label_list[i]] for i in range(num_labels)}
        else:
            logger.warn(
                "Your model seems to have been trained with labels, but they don't match the dataset: ",
                f"model labels: {list(sorted(label_name_to_id.keys()))}, dataset labels: {list(sorted(label_list))}."
                "\nIgnoring the model labels as a result.",
            )
    elif data_args.task_name is None:
        label_to_id = {v: i for i, v in enumerate(label_list)}
293

Sylvain Gugger's avatar
Sylvain Gugger committed
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
    def preprocess_function(examples):
        # Tokenize the texts
        args = (
            (examples[sentence1_key],) if sentence2_key is None else (examples[sentence1_key], examples[sentence2_key])
        )
        result = tokenizer(*args, padding=padding, max_length=max_length, truncation=True)

        # Map labels to IDs (not necessary for GLUE tasks)
        if label_to_id is not None and "label" in examples:
            result["label"] = [label_to_id[l] for l in examples["label"]]
        return result

    datasets = datasets.map(preprocess_function, batched=True, load_from_cache_file=not data_args.overwrite_cache)

    train_dataset = datasets["train"]
    eval_dataset = datasets["validation_matched" if data_args.task_name == "mnli" else "validation"]
    if data_args.task_name is not None:
        test_dataset = datasets["test_matched" if data_args.task_name == "mnli" else "test"]

    # Log a few random samples from the training set:
    for index in random.sample(range(len(train_dataset)), 3):
        logger.info(f"Sample {index} of the training set: {train_dataset[index]}.")

    # Get the metric function
    if data_args.task_name is not None:
        metric = load_metric("glue", data_args.task_name)
    # TODO: When datasets metrics include regular accuracy, make an else here and remove special branch from
    # compute_metrics

    # You can define your custom compute_metrics function. It takes an `EvalPrediction` object (a namedtuple with a
    # predictions and label_ids field) and has to return a dictionary string to float.
    def compute_metrics(p: EvalPrediction):
        preds = p.predictions[0] if isinstance(p.predictions, tuple) else p.predictions
        preds = np.squeeze(preds) if is_regression else np.argmax(preds, axis=1)
        if data_args.task_name is not None:
            result = metric.compute(predictions=preds, references=p.label_ids)
            if len(result) > 1:
                result["combined_score"] = np.mean(list(result.values())).item()
            return result
        elif is_regression:
            return {"mse": ((preds - p.label_ids) ** 2).mean().item()}
        else:
            return {"accuracy": (preds == p.label_ids).astype(np.float32).mean().item()}
thomwolf's avatar
thomwolf committed
337

Julien Chaumond's avatar
Julien Chaumond committed
338
339
340
341
342
    # Initialize our Trainer
    trainer = Trainer(
        model=model,
        args=training_args,
        train_dataset=train_dataset,
Sylvain Gugger's avatar
Sylvain Gugger committed
343
344
345
346
347
        eval_dataset=eval_dataset if training_args.do_eval else None,
        compute_metrics=compute_metrics,
        tokenizer=tokenizer,
        # Data collator will default to DataCollatorWithPadding, so we change it if we already did the padding.
        data_collator=default_data_collator if data_args.pad_to_max_length else None,
Julien Chaumond's avatar
Julien Chaumond committed
348
    )
thomwolf's avatar
thomwolf committed
349

thomwolf's avatar
thomwolf committed
350
    # Training
Julien Chaumond's avatar
Julien Chaumond committed
351
352
353
354
    if training_args.do_train:
        trainer.train(
            model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None
        )
Sylvain Gugger's avatar
Sylvain Gugger committed
355
        trainer.save_model()  # Saves the tokenizer too for easy upload
thomwolf's avatar
thomwolf committed
356

thomwolf's avatar
thomwolf committed
357
    # Evaluation
358
    eval_results = {}
359
    if training_args.do_eval:
Julien Chaumond's avatar
Julien Chaumond committed
360
361
362
        logger.info("*** Evaluate ***")

        # Loop to handle MNLI double evaluation (matched, mis-matched)
Sylvain Gugger's avatar
Sylvain Gugger committed
363
        tasks = [data_args.task_name]
Julien Chaumond's avatar
Julien Chaumond committed
364
365
        eval_datasets = [eval_dataset]
        if data_args.task_name == "mnli":
Sylvain Gugger's avatar
Sylvain Gugger committed
366
367
            tasks.append("mnli-mm")
            eval_datasets.append(datasets["validation_mismatched"])
Julien Chaumond's avatar
Julien Chaumond committed
368

Sylvain Gugger's avatar
Sylvain Gugger committed
369
        for eval_dataset, task in zip(eval_datasets, tasks):
370
            eval_result = trainer.evaluate(eval_dataset=eval_dataset)
Julien Chaumond's avatar
Julien Chaumond committed
371

Sylvain Gugger's avatar
Sylvain Gugger committed
372
373
            output_eval_file = os.path.join(training_args.output_dir, f"eval_results_{task}.txt")
            if trainer.is_world_process_zero():
374
                with open(output_eval_file, "w") as writer:
Sylvain Gugger's avatar
Sylvain Gugger committed
375
                    logger.info(f"***** Eval results {task} *****")
376
                    for key, value in eval_result.items():
Sylvain Gugger's avatar
Sylvain Gugger committed
377
378
                        logger.info(f"  {key} = {value}")
                        writer.write(f"{key} = {value}\n")
379

380
            eval_results.update(eval_result)
thomwolf's avatar
thomwolf committed
381

382
    if training_args.do_predict:
Sylvain Gugger's avatar
Sylvain Gugger committed
383
384
385
386
        logger.info("*** Test ***")

        # Loop to handle MNLI double evaluation (matched, mis-matched)
        tasks = [data_args.task_name]
387
388
        test_datasets = [test_dataset]
        if data_args.task_name == "mnli":
Sylvain Gugger's avatar
Sylvain Gugger committed
389
390
            tasks.append("mnli-mm")
            test_datasets.append(datasets["test_mismatched"])
391

Sylvain Gugger's avatar
Sylvain Gugger committed
392
393
394
        for test_dataset, task in zip(test_datasets, tasks):
            # Removing the `label` columns because it contains -1 and Trainer won't like that.
            test_dataset.remove_columns_("label")
395
            predictions = trainer.predict(test_dataset=test_dataset).predictions
Sylvain Gugger's avatar
Sylvain Gugger committed
396
            predictions = np.squeeze(predictions) if is_regression else np.argmax(predictions, axis=1)
397

Sylvain Gugger's avatar
Sylvain Gugger committed
398
399
            output_test_file = os.path.join(training_args.output_dir, f"test_results_{task}.txt")
            if trainer.is_world_process_zero():
400
                with open(output_test_file, "w") as writer:
Sylvain Gugger's avatar
Sylvain Gugger committed
401
                    logger.info(f"***** Test results {task} *****")
402
403
                    writer.write("index\tprediction\n")
                    for index, item in enumerate(predictions):
Sylvain Gugger's avatar
Sylvain Gugger committed
404
405
                        if is_regression:
                            writer.write(f"{index}\t{item:3.3f}\n")
406
                        else:
Sylvain Gugger's avatar
Sylvain Gugger committed
407
408
                            item = label_list[item]
                            writer.write(f"{index}\t{item}\n")
409
    return eval_results
thomwolf's avatar
thomwolf committed
410
411


Lysandre Debut's avatar
Lysandre Debut committed
412
413
414
415
416
def _mp_fn(index):
    # For xla_spawn (TPUs)
    main()


thomwolf's avatar
thomwolf committed
417
418
if __name__ == "__main__":
    main()