run_xlnet_classifier.py 24.5 KB
Newer Older
1
# coding=utf-8
thomwolf's avatar
thomwolf committed
2
# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
3
# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
4
5
6
7
8
9
10
11
12
13
14
15
16
17
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""BERT finetuning runner."""

thomwolf's avatar
thomwolf committed
18
from __future__ import absolute_import, division, print_function
19

thomwolf's avatar
thomwolf committed
20
import argparse
21
import logging
thomwolf's avatar
thomwolf committed
22
import os
thomwolf's avatar
thomwolf committed
23
import sys
VictorSanh's avatar
VictorSanh committed
24
import random
25
from tqdm import tqdm, trange
thomwolf's avatar
thomwolf committed
26
27

import numpy as np
28

VictorSanh's avatar
VictorSanh committed
29
import torch
thomwolf's avatar
thomwolf committed
30
31
from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler,
                              TensorDataset)
32
from torch.utils.data.distributed import DistributedSampler
33
34
from torch.nn import CrossEntropyLoss, MSELoss

35
36
37
from tensorboardX import SummaryWriter

from pytorch_pretrained_bert.file_utils import WEIGHTS_NAME, CONFIG_NAME
38
39
from pytorch_pretrained_bert.modeling_xlnet import XLNetForSequenceClassification
from pytorch_pretrained_bert.tokenization_xlnet import XLNetTokenizer
40
from pytorch_pretrained_bert.optimization import BertAdam, WarmupLinearSchedule
41

42
from utils_glue import processors, output_modes, convert_examples_to_features, compute_metrics
43

thomwolf's avatar
thomwolf committed
44
45
46
47
48
49
if sys.version_info[0] == 2:
    import cPickle as pickle
else:
    import pickle


50
logger = logging.getLogger(__name__)
51

VictorSanh's avatar
WIP  
VictorSanh committed
52

53
def main():
54
55
56
    parser = argparse.ArgumentParser()

    ## Required parameters
thomwolf's avatar
thomwolf committed
57
    parser.add_argument("--data_dir", default=None, type=str, required=True,
58
                        help="The input data dir. Should contain the .tsv files (or other data files) for the task.")
thomwolf's avatar
thomwolf committed
59
    parser.add_argument("--task_name", default=None, type=str, required=True,
60
                        help="The name of the task to train.")
thomwolf's avatar
thomwolf committed
61
    parser.add_argument("--output_dir", default=None, type=str, required=True,
62
                        help="The output directory where the model predictions and checkpoints will be written.")
thomwolf's avatar
thomwolf committed
63
64
    # training
    parser.add_argument("--do_train", action='store_true',
65
                        help="Whether to run training.")
thomwolf's avatar
thomwolf committed
66
    parser.add_argument("--learning_rate", default=5e-5, type=float,
67
                        help="The initial learning rate for Adam.")
thomwolf's avatar
thomwolf committed
68
    parser.add_argument("--num_train_epochs", default=3.0, type=float,
69
                        help="Total number of training epochs to perform.")
thomwolf's avatar
thomwolf committed
70
    parser.add_argument("--warmup_proportion", default=0.1, type=float,
71
72
                        help="Proportion of training to perform linear learning rate warmup for. "
                             "E.g., 0.1 = 10%% of training.")
thomwolf's avatar
thomwolf committed
73
74
75
    parser.add_argument("--train_batch_size", default=32, type=int,
                        help="Total batch size for training.")
    parser.add_argument('--gradient_accumulation_steps', type=int, default=1,
76
                        help="Number of updates steps to accumulate before performing a backward/update pass.")
thomwolf's avatar
thomwolf committed
77
    parser.add_argument('--fp16', action='store_true',
thomwolf's avatar
thomwolf committed
78
                        help="Whether to use 16-bit float precision instead of 32-bit")
thomwolf's avatar
thomwolf committed
79
    parser.add_argument('--loss_scale', type=float, default=0,
80
81
82
                        help="Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
                             "0 (default value): dynamic loss scaling.\n"
                             "Positive power of 2: static loss scaling value.\n")
thomwolf's avatar
thomwolf committed
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
    # evaluation
    parser.add_argument("--do_eval", action='store_true',
                        help="Whether to run eval on the dev set.")
    parser.add_argument("--eval_batch_size", default=8, type=int,
                        help="Total batch size for eval.")
    # Model
    parser.add_argument("--xlnet_model", default="xlnet-large-cased", type=str,
                        help="XLNet pre-trained model: currently only xlnet-large-cased.")
    parser.add_argument("--do_lower_case", action='store_true',
                        help="Set this flag if you are using an uncased model.")
    parser.add_argument("--cache_dir", default="", type=str,
                        help="Where do you want to store the pre-trained models downloaded from s3")
    # task specific
    parser.add_argument("--max_seq_length", default=128, type=int,
                        help="The maximum total input sequence length after WordPiece tokenization. \n"
                             "Sequences longer than this will be truncated, and sequences shorter \n"
                             "than this will be padded.")
    parser.add_argument('--overwrite_output_dir', action='store_true',
                        help="Overwrite the content of the output directory")
    # Misc
    parser.add_argument("--no_cuda", action='store_true',
                        help="Whether not to use CUDA when available")
    parser.add_argument("--local_rank", type=int, default=-1,
                        help="local_rank for distributed training on gpus")
    parser.add_argument('--seed', type=int, default=42,
                        help="random seed for initialization")
109
110
    parser.add_argument('--server_ip', type=str, default='', help="Can be used for distant debugging.")
    parser.add_argument('--server_port', type=str, default='', help="Can be used for distant debugging.")
111
112
    args = parser.parse_args()

113
114
115
116
117
118
119
    if args.server_ip and args.server_port:
        # Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
        import ptvsd
        print("Waiting for debugger attach")
        ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True)
        ptvsd.wait_for_attach()

thomwolf's avatar
thomwolf committed
120
121
122
123
    if args.local_rank == -1 or args.no_cuda:
        device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
        n_gpu = torch.cuda.device_count()
    else:
124
        torch.cuda.set_device(args.local_rank)
thomwolf's avatar
thomwolf committed
125
126
        device = torch.device("cuda", args.local_rank)
        n_gpu = 1
thomwolf's avatar
thomwolf committed
127
128
        # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
        torch.distributed.init_process_group(backend='nccl')
129
    args.device = device
130
131
132
133
134

    logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
                        datefmt = '%m/%d/%Y %H:%M:%S',
                        level = logging.INFO if args.local_rank in [-1, 0] else logging.WARN)

135
136
    logger.info("device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".format(
        device, n_gpu, bool(args.local_rank != -1), args.fp16))
thomwolf's avatar
thomwolf committed
137

138
139
140
    if args.gradient_accumulation_steps < 1:
        raise ValueError("Invalid gradient_accumulation_steps parameter: {}, should be >= 1".format(
                            args.gradient_accumulation_steps))
thomwolf's avatar
thomwolf committed
141

142
    args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps
thomwolf's avatar
thomwolf committed
143

VictorSanh's avatar
VictorSanh committed
144
145
146
    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
thomwolf's avatar
thomwolf committed
147
148
    if n_gpu > 0:
        torch.cuda.manual_seed_all(args.seed)
thomwolf's avatar
thomwolf committed
149

VictorSanh's avatar
WIP  
VictorSanh committed
150
151
    if not args.do_train and not args.do_eval:
        raise ValueError("At least one of `do_train` or `do_eval` must be True.")
thomwolf's avatar
thomwolf committed
152

thomwolf's avatar
thomwolf committed
153
    if os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train and not args.overwrite_output_dir:
thomwolf's avatar
thomwolf committed
154
        raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir))
thomwolf's avatar
thomwolf committed
155
    if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
thomwolf's avatar
thomwolf committed
156
        os.makedirs(args.output_dir)
VictorSanh's avatar
WIP  
VictorSanh committed
157
158

    task_name = args.task_name.lower()
thomwolf's avatar
thomwolf committed
159

VictorSanh's avatar
WIP  
VictorSanh committed
160
161
162
163
    if task_name not in processors:
        raise ValueError("Task not found: %s" % (task_name))

    processor = processors[task_name]()
164
165
    output_mode = output_modes[task_name]

VictorSanh's avatar
WIP  
VictorSanh committed
166
    label_list = processor.get_labels()
167
    num_labels = len(label_list)
VictorSanh's avatar
WIP  
VictorSanh committed
168

thomwolf's avatar
thomwolf committed
169
    if args.local_rank not in [-1, 0]:
thomwolf's avatar
thomwolf committed
170
        torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab
171
172
    tokenizer = XLNetTokenizer.from_pretrained(args.xlnet_model, do_lower_case=args.do_lower_case)
    model = XLNetForSequenceClassification.from_pretrained(args.xlnet_model, num_labels=num_labels)
thomwolf's avatar
thomwolf committed
173
    if args.local_rank == 0:
thomwolf's avatar
thomwolf committed
174
        torch.distributed.barrier()
thomwolf's avatar
thomwolf committed
175

samuel.broscheit's avatar
samuel.broscheit committed
176
177
178
179
    if args.fp16:
        model.half()
    model.to(device)
    if args.local_rank != -1:
180
181
182
183
        model = torch.nn.parallel.DistributedDataParallel(model,
                                                          device_ids=[args.local_rank],
                                                          output_device=args.local_rank,
                                                          find_unused_parameters=True)
samuel.broscheit's avatar
samuel.broscheit committed
184
185
186
    elif n_gpu > 1:
        model = torch.nn.DataParallel(model)

thomwolf's avatar
fixing  
thomwolf committed
187
188
189
190
    global_step = 0
    nb_tr_steps = 0
    tr_loss = 0

VictorSanh's avatar
WIP  
VictorSanh committed
191
    if args.do_train:
192
193
        if args.local_rank in [-1, 0]:
            tb_writer = SummaryWriter()
samuel.broscheit's avatar
samuel.broscheit committed
194
195

        # Prepare data loader
VictorSanh's avatar
WIP  
VictorSanh committed
196
        train_examples = processor.get_train_examples(args.data_dir)
thomwolf's avatar
thomwolf committed
197
        cached_train_features_file = os.path.join(args.data_dir, 'train_{0}_{1}_{2}'.format(
198
            list(filter(None, args.xlnet_model.split('/'))).pop(),
199
                        str(args.max_seq_length),
thomwolf's avatar
thomwolf committed
200
                        str(task_name)))
201
202
203
204
205
206
207
208
209
210
211
        try:
            with open(cached_train_features_file, "rb") as reader:
                train_features = pickle.load(reader)
        except:
            train_features = convert_examples_to_features(
                train_examples, label_list, args.max_seq_length, tokenizer, output_mode)
            if args.local_rank == -1 or torch.distributed.get_rank() == 0:
                logger.info("  Saving train features into cached file %s", cached_train_features_file)
                with open(cached_train_features_file, "wb") as writer:
                    pickle.dump(train_features, writer)

212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
        all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long)
        all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long)
        all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long)

        if output_mode == "classification":
            all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long)
        elif output_mode == "regression":
            all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.float)

        train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
        if args.local_rank == -1:
            train_sampler = RandomSampler(train_data)
        else:
            train_sampler = DistributedSampler(train_data)
        train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size)

228
        num_train_optimization_steps = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs
thomwolf's avatar
thomwolf committed
229

samuel.broscheit's avatar
samuel.broscheit committed
230
        # Prepare optimizer
thomwolf's avatar
thomwolf committed
231

232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
        param_optimizer = list(model.named_parameters())
        no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
        optimizer_grouped_parameters = [
            {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},
            {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
            ]
        if args.fp16:
            try:
                from apex.optimizers import FP16_Optimizer
                from apex.optimizers import FusedAdam
            except ImportError:
                raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.")

            optimizer = FusedAdam(optimizer_grouped_parameters,
                                  lr=args.learning_rate,
                                  bias_correction=False,
                                  max_grad_norm=1.0)
            if args.loss_scale == 0:
                optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True)
            else:
                optimizer = FP16_Optimizer(optimizer, static_loss_scale=args.loss_scale)
            warmup_linear = WarmupLinearSchedule(warmup=args.warmup_proportion,
                                                 t_total=num_train_optimization_steps)
thomwolf's avatar
thomwolf committed
255

256
        else:
257
258
259
260
            optimizer = BertAdam(optimizer_grouped_parameters,
                                 lr=args.learning_rate,
                                 warmup=args.warmup_proportion,
                                 t_total=num_train_optimization_steps)
thomwolf's avatar
thomwolf committed
261

VictorSanh's avatar
WIP  
VictorSanh committed
262
263
264
        logger.info("***** Running training *****")
        logger.info("  Num examples = %d", len(train_examples))
        logger.info("  Batch size = %d", args.train_batch_size)
265
        logger.info("  Num steps = %d", num_train_optimization_steps)
266
267

        model.train()
thomwolf's avatar
thomwolf committed
268
        for _ in trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0]):
269
270
            tr_loss = 0
            nb_tr_examples, nb_tr_steps = 0, 0
thomwolf's avatar
thomwolf committed
271
            for step, batch in enumerate(tqdm(train_dataloader, desc="Iteration", disable=args.local_rank not in [-1, 0])):
272
273
                batch = tuple(t.to(device) for t in batch)
                input_ids, input_mask, segment_ids, label_ids = batch
274
275

                # define a new function to compute loss values for both output_modes
thomwolf's avatar
thomwolf committed
276
                logits, _ = model(input_ids, token_type_ids=segment_ids, attention_mask=input_mask)
277
278
279
280
281
282
283
284

                if output_mode == "classification":
                    loss_fct = CrossEntropyLoss()
                    loss = loss_fct(logits.view(-1, num_labels), label_ids.view(-1))
                elif output_mode == "regression":
                    loss_fct = MSELoss()
                    loss = loss_fct(logits.view(-1), label_ids.view(-1))

thomwolf's avatar
thomwolf committed
285
286
                if n_gpu > 1:
                    loss = loss.mean() # mean() to average on multi-gpu.
287
288
                if args.gradient_accumulation_steps > 1:
                    loss = loss / args.gradient_accumulation_steps
289
290
291
292
293
294

                if args.fp16:
                    optimizer.backward(loss)
                else:
                    loss.backward()

295
                tr_loss += loss.item()
296
                nb_tr_examples += input_ids.size(0)
297
                nb_tr_steps += 1
thomwolf's avatar
thomwolf committed
298
                if (step + 1) % args.gradient_accumulation_steps == 0:
299
300
301
                    if args.fp16:
                        # modify learning rate with special warm up BERT uses
                        # if args.fp16 is False, BertAdam is used that handles this automatically
302
                        lr_this_step = args.learning_rate * warmup_linear.get_lr(global_step, args.warmup_proportion)
303
304
                        for param_group in optimizer.param_groups:
                            param_group['lr'] = lr_this_step
305
306
                    optimizer.step()
                    optimizer.zero_grad()
thomwolf's avatar
thomwolf committed
307
                    global_step += 1
308
309
310
                    if args.local_rank in [-1, 0]:
                        tb_writer.add_scalar('lr', optimizer.get_lr()[0], global_step)
                        tb_writer.add_scalar('loss', loss.item(), global_step)
thomwolf's avatar
thomwolf committed
311

312
313
    ### Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()
    ### Example:
314
    if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0):
315
        # Save a trained model, configuration and tokenizer
316
        model_to_save = model.module if hasattr(model, 'module') else model  # Only save the model it-self
317
318

        # If we save using the predefined names, we can load using `from_pretrained`
319
320
321
        output_model_file = os.path.join(args.output_dir, WEIGHTS_NAME)
        output_config_file = os.path.join(args.output_dir, CONFIG_NAME)

322
323
        torch.save(model_to_save.state_dict(), output_model_file)
        model_to_save.config.to_json_file(output_config_file)
324
        tokenizer.save_vocabulary(args.output_dir)
325

326
        # Load a trained model and vocabulary that you have fine-tuned
327
328
        model = XLNetForSequenceClassification.from_pretrained(args.output_dir, num_labels=num_labels)
        tokenizer = XLNetTokenizer.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
thomwolf's avatar
thomwolf committed
329
330
331
332

        # Good practice: save your training arguments together with the trained model
        output_args_file = os.path.join(args.output_dir, 'training_args.bin')
        torch.save(args, output_args_file)
thomwolf's avatar
thomwolf committed
333
    else:
334
        model = XLNetForSequenceClassification.from_pretrained(args.xlnet_model, num_labels=num_labels)
335

thomwolf's avatar
thomwolf committed
336
    model.to(device)
337
338

    ### Evaluation
thomwolf's avatar
thomwolf committed
339
    if args.do_eval and (args.local_rank == -1 or torch.distributed.get_rank() == 0):
VictorSanh's avatar
WIP  
VictorSanh committed
340
        eval_examples = processor.get_dev_examples(args.data_dir)
thomwolf's avatar
thomwolf committed
341
        cached_eval_features_file = os.path.join(args.data_dir, 'dev_{0}_{1}_{2}'.format(
342
            list(filter(None, args.xlnet_model.split('/'))).pop(),
thomwolf's avatar
thomwolf committed
343
344
345
                        str(args.max_seq_length),
                        str(task_name)))
        try:
thomwolf's avatar
thomwolf committed
346
            with open(cached_eval_features_file, "rb") as reader:
thomwolf's avatar
thomwolf committed
347
                eval_features = pickle.load(reader)
thomwolf's avatar
thomwolf committed
348
        except:
thomwolf's avatar
thomwolf committed
349
350
            eval_features = convert_examples_to_features(
                eval_examples, label_list, args.max_seq_length, tokenizer, output_mode)
thomwolf's avatar
thomwolf committed
351
            if args.local_rank == -1 or torch.distributed.get_rank() == 0:
thomwolf's avatar
thomwolf committed
352
353
354
                logger.info("  Saving eval features into cached file %s", cached_eval_features_file)
                with open(cached_eval_features_file, "wb") as writer:
                    pickle.dump(eval_features, writer)
thomwolf's avatar
thomwolf committed
355
356


VictorSanh's avatar
wip  
VictorSanh committed
357
358
359
        logger.info("***** Running evaluation *****")
        logger.info("  Num examples = %d", len(eval_examples))
        logger.info("  Batch size = %d", args.eval_batch_size)
360
361
362
        all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long)
        all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long)
        all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long)
363
364
365
366
367
368

        if output_mode == "classification":
            all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long)
        elif output_mode == "regression":
            all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.float)

369
        eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
370
        # Run prediction for full data
371
372
373
374
        if args.local_rank == -1:
            eval_sampler = SequentialSampler(eval_data)
        else:
            eval_sampler = DistributedSampler(eval_data)  # Note that this sampler samples randomly
375
376
377
        eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size)

        model.eval()
378
379
380
        eval_loss = 0
        nb_eval_steps = 0
        preds = []
thomwolf's avatar
thomwolf committed
381
        out_label_ids = None
382

383
        for input_ids, input_mask, segment_ids, label_ids in tqdm(eval_dataloader, desc="Evaluating"):
384
            input_ids = input_ids.to(device)
thomwolf's avatar
thomwolf committed
385
            input_mask = input_mask.to(device)
386
            segment_ids = segment_ids.to(device)
387
            label_ids = label_ids.to(device)
388

389
            with torch.no_grad():
thomwolf's avatar
thomwolf committed
390
                logits, _ = model(input_ids, token_type_ids=segment_ids, attention_mask=input_mask)
391

392
393
394
395
396
397
398
            # create eval loss and other metric required by the task
            if output_mode == "classification":
                loss_fct = CrossEntropyLoss()
                tmp_eval_loss = loss_fct(logits.view(-1, num_labels), label_ids.view(-1))
            elif output_mode == "regression":
                loss_fct = MSELoss()
                tmp_eval_loss = loss_fct(logits.view(-1), label_ids.view(-1))
399

400
401
            eval_loss += tmp_eval_loss.mean().item()
            nb_eval_steps += 1
402
403
            if len(preds) == 0:
                preds.append(logits.detach().cpu().numpy())
thomwolf's avatar
hop  
thomwolf committed
404
                out_label_ids = label_ids.detach().cpu().numpy()
405
406
407
            else:
                preds[0] = np.append(
                    preds[0], logits.detach().cpu().numpy(), axis=0)
thomwolf's avatar
thomwolf committed
408
409
                out_label_ids = np.append(
                    out_label_ids, label_ids.detach().cpu().numpy(), axis=0)
VictorSanh's avatar
WIP  
VictorSanh committed
410

411
        eval_loss = eval_loss / nb_eval_steps
412
413
414
        preds = preds[0]
        if output_mode == "classification":
            preds = np.argmax(preds, axis=1)
415
416
        elif output_mode == "regression":
            preds = np.squeeze(preds)
thomwolf's avatar
thomwolf committed
417
        result = compute_metrics(task_name, preds, out_label_ids)
418

419
        loss = tr_loss/global_step if args.do_train else None
420
421
422
423

        result['eval_loss'] = eval_loss
        result['global_step'] = global_step
        result['loss'] = loss
VictorSanh's avatar
WIP  
VictorSanh committed
424
425

        output_eval_file = os.path.join(args.output_dir, "eval_results.txt")
VictorSanh's avatar
wip  
VictorSanh committed
426
427
        with open(output_eval_file, "w") as writer:
            logger.info("***** Eval results *****")
VictorSanh's avatar
WIP  
VictorSanh committed
428
            for key in sorted(result.keys()):
VictorSanh's avatar
wip  
VictorSanh committed
429
                logger.info("  %s = %s", key, str(result[key]))
VictorSanh's avatar
WIP  
VictorSanh committed
430
                writer.write("%s = %s\n" % (key, str(result[key])))
431

432
433
434
435
436
        # hack for MNLI-MM
        if task_name == "mnli":
            task_name = "mnli-mm"
            processor = processors[task_name]()

437
438
439
440
441
            if os.path.exists(args.output_dir + '-MM') and os.listdir(args.output_dir + '-MM') and args.do_train:
                raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir))
            if not os.path.exists(args.output_dir + '-MM'):
                os.makedirs(args.output_dir + '-MM')

442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
            eval_examples = processor.get_dev_examples(args.data_dir)
            eval_features = convert_examples_to_features(
                eval_examples, label_list, args.max_seq_length, tokenizer, output_mode)
            logger.info("***** Running evaluation *****")
            logger.info("  Num examples = %d", len(eval_examples))
            logger.info("  Batch size = %d", args.eval_batch_size)
            all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long)
            all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long)
            all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long)
            all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long)

            eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
            # Run prediction for full data
            eval_sampler = SequentialSampler(eval_data)
            eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size)

            model.eval()
            eval_loss = 0
            nb_eval_steps = 0
            preds = []
thomwolf's avatar
thomwolf committed
462
            out_label_ids = None
463
464
465
466
467
468
469
470

            for input_ids, input_mask, segment_ids, label_ids in tqdm(eval_dataloader, desc="Evaluating"):
                input_ids = input_ids.to(device)
                input_mask = input_mask.to(device)
                segment_ids = segment_ids.to(device)
                label_ids = label_ids.to(device)

                with torch.no_grad():
thomwolf's avatar
thomwolf committed
471
                    logits, _ = model(input_ids, token_type_ids=segment_ids, attention_mask=input_mask, labels=None)
thomwolf's avatar
thomwolf committed
472

473
474
                loss_fct = CrossEntropyLoss()
                tmp_eval_loss = loss_fct(logits.view(-1, num_labels), label_ids.view(-1))
thomwolf's avatar
thomwolf committed
475

476
477
478
479
                eval_loss += tmp_eval_loss.mean().item()
                nb_eval_steps += 1
                if len(preds) == 0:
                    preds.append(logits.detach().cpu().numpy())
thomwolf's avatar
hop  
thomwolf committed
480
                    out_label_ids = label_ids.detach().cpu().numpy()
481
482
483
                else:
                    preds[0] = np.append(
                        preds[0], logits.detach().cpu().numpy(), axis=0)
thomwolf's avatar
thomwolf committed
484
485
                    out_label_ids = np.append(
                        out_label_ids, label_ids.detach().cpu().numpy(), axis=0)
486

487
488
489
            eval_loss = eval_loss / nb_eval_steps
            preds = preds[0]
            preds = np.argmax(preds, axis=1)
thomwolf's avatar
thomwolf committed
490
            result = compute_metrics(task_name, preds, out_label_ids)
491

492
            loss = tr_loss/global_step if args.do_train else None
493
494
495
496
497
498
499
500
501
502
503

            result['eval_loss'] = eval_loss
            result['global_step'] = global_step
            result['loss'] = loss

            output_eval_file = os.path.join(args.output_dir + '-MM', "eval_results.txt")
            with open(output_eval_file, "w") as writer:
                logger.info("***** Eval results *****")
                for key in sorted(result.keys()):
                    logger.info("  %s = %s", key, str(result[key]))
                    writer.write("%s = %s\n" % (key, str(result[key])))
504

VictorSanh's avatar
WIP  
VictorSanh committed
505
506
if __name__ == "__main__":
    main()