training_args.py 44.3 KB
Newer Older
Sylvain Gugger's avatar
Sylvain Gugger committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2020 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

Julien Chaumond's avatar
Julien Chaumond committed
15
import json
Julien Plu's avatar
Julien Plu committed
16
import os
17
import warnings
18
from dataclasses import asdict, dataclass, field
19
from enum import Enum
20
from typing import Any, Dict, List, Optional
Julien Chaumond's avatar
Julien Chaumond committed
21

Sylvain Gugger's avatar
Sylvain Gugger committed
22
23
from .file_utils import (
    cached_property,
Sylvain Gugger's avatar
Sylvain Gugger committed
24
25
    is_sagemaker_dp_enabled,
    is_sagemaker_mp_enabled,
Sylvain Gugger's avatar
Sylvain Gugger committed
26
27
28
29
    is_torch_available,
    is_torch_tpu_available,
    torch_required,
)
30
from .trainer_utils import EvaluationStrategy, IntervalStrategy, SchedulerType, ShardedDDPOption
Lysandre Debut's avatar
Lysandre Debut committed
31
from .utils import logging
Julien Chaumond's avatar
Julien Chaumond committed
32
33
34
35
36


if is_torch_available():
    import torch

37
if is_torch_tpu_available():
Lysandre Debut's avatar
Lysandre Debut committed
38
39
    import torch_xla.core.xla_model as xm

Sylvain Gugger's avatar
Sylvain Gugger committed
40
if is_sagemaker_dp_enabled():
41
42
    import smdistributed.dataparallel.torch.distributed as sm_dist

Sylvain Gugger's avatar
Sylvain Gugger committed
43
44
45
46
47
if is_sagemaker_mp_enabled():
    import smdistributed.modelparallel.torch as smp

    smp.init()

Lysandre Debut's avatar
Lysandre Debut committed
48

Lysandre Debut's avatar
Lysandre Debut committed
49
logger = logging.get_logger(__name__)
50
51


Julien Plu's avatar
Julien Plu committed
52
53
54
55
56
57
58
59
60
61
62
def default_logdir() -> str:
    """
    Same default as PyTorch
    """
    import socket
    from datetime import datetime

    current_time = datetime.now().strftime("%b%d_%H-%M-%S")
    return os.path.join("runs", current_time + "_" + socket.gethostname())


63
64
65
@dataclass
class TrainingArguments:
    """
Sylvain Gugger's avatar
Sylvain Gugger committed
66
67
    TrainingArguments is the subset of the arguments we use in our example scripts **which relate to the training loop
    itself**.
68

69
70
71
    Using :class:`~transformers.HfArgumentParser` we can turn this class into `argparse
    <https://docs.python.org/3/library/argparse.html#module-argparse>`__ arguments that can be specified on the command
    line.
72
73
74
75
76
77
78
79

    Parameters:
        output_dir (:obj:`str`):
            The output directory where the model predictions and checkpoints will be written.
        overwrite_output_dir (:obj:`bool`, `optional`, defaults to :obj:`False`):
            If :obj:`True`, overwrite the content of the output directory. Use this to continue training if
            :obj:`output_dir` points to a checkpoint directory.
        do_train (:obj:`bool`, `optional`, defaults to :obj:`False`):
80
81
82
            Whether to run training or not. This argument is not directly used by :class:`~transformers.Trainer`, it's
            intended to be used by your training/evaluation scripts instead. See the `example scripts
            <https://github.com/huggingface/transformers/tree/master/examples>`__ for more details.
83
        do_eval (:obj:`bool`, `optional`):
84
85
86
87
88
            Whether to run evaluation on the validation set or not. Will be set to :obj:`True` if
            :obj:`evaluation_strategy` is different from :obj:`"no"`. This argument is not directly used by
            :class:`~transformers.Trainer`, it's intended to be used by your training/evaluation scripts instead. See
            the `example scripts <https://github.com/huggingface/transformers/tree/master/examples>`__ for more
            details.
89
        do_predict (:obj:`bool`, `optional`, defaults to :obj:`False`):
90
91
92
93
            Whether to run predictions on the test set or not. This argument is not directly used by
            :class:`~transformers.Trainer`, it's intended to be used by your training/evaluation scripts instead. See
            the `example scripts <https://github.com/huggingface/transformers/tree/master/examples>`__ for more
            details.
94
        evaluation_strategy (:obj:`str` or :class:`~transformers.trainer_utils.IntervalStrategy`, `optional`, defaults to :obj:`"no"`):
95
96
97
98
99
100
            The evaluation strategy to adopt during training. Possible values are:

                * :obj:`"no"`: No evaluation is done during training.
                * :obj:`"steps"`: Evaluation is done (and logged) every :obj:`eval_steps`.
                * :obj:`"epoch"`: Evaluation is done at the end of each epoch.

101
        prediction_loss_only (:obj:`bool`, `optional`, defaults to `False`):
102
            When performing evaluation and generating predictions, only returns the loss.
103
104
105
106
        per_device_train_batch_size (:obj:`int`, `optional`, defaults to 8):
            The batch size per GPU/TPU core/CPU for training.
        per_device_eval_batch_size (:obj:`int`, `optional`, defaults to 8):
            The batch size per GPU/TPU core/CPU for evaluation.
107
        gradient_accumulation_steps (:obj:`int`, `optional`, defaults to 1):
108
            Number of updates steps to accumulate the gradients for, before performing a backward/update pass.
109
110
111
112
113
114

            .. warning::

                When using gradient accumulation, one step is counted as one step with backward pass. Therefore,
                logging, evaluation, save will be conducted every ``gradient_accumulation_steps * xxx_step`` training
                examples.
115
116
117
118
        eval_accumulation_steps (:obj:`int`, `optional`):
            Number of predictions steps to accumulate the output tensors for, before moving the results to the CPU. If
            left unset, the whole predictions are accumulated on GPU/TPU before being moved to the CPU (faster but
            requires more memory).
119
        learning_rate (:obj:`float`, `optional`, defaults to 5e-5):
120
            The initial learning rate for :class:`~transformers.AdamW` optimizer.
121
        weight_decay (:obj:`float`, `optional`, defaults to 0):
Sylvain Gugger's avatar
Sylvain Gugger committed
122
            The weight decay to apply (if not zero) to all layers except all bias and LayerNorm weights in
123
            :class:`~transformers.AdamW` optimizer.
124
        adam_beta1 (:obj:`float`, `optional`, defaults to 0.9):
125
            The beta1 hyperparameter for the :class:`~transformers.AdamW` optimizer.
126
        adam_beta2 (:obj:`float`, `optional`, defaults to 0.999):
127
            The beta2 hyperparameter for the :class:`~transformers.AdamW` optimizer.
128
        adam_epsilon (:obj:`float`, `optional`, defaults to 1e-8):
129
            The epsilon hyperparameter for the :class:`~transformers.AdamW` optimizer.
130
131
132
        max_grad_norm (:obj:`float`, `optional`, defaults to 1.0):
            Maximum gradient norm (for gradient clipping).
        num_train_epochs(:obj:`float`, `optional`, defaults to 3.0):
Sylvain Gugger's avatar
Sylvain Gugger committed
133
134
            Total number of training epochs to perform (if not an integer, will perform the decimal part percents of
            the last epoch before stopping training).
135
136
137
        max_steps (:obj:`int`, `optional`, defaults to -1):
            If set to a positive number, the total number of training steps to perform. Overrides
            :obj:`num_train_epochs`.
Sylvain Gugger's avatar
Sylvain Gugger committed
138
139
140
        lr_scheduler_type (:obj:`str` or :class:`~transformers.SchedulerType`, `optional`, defaults to :obj:`"linear"`):
            The scheduler type to use. See the documentation of :class:`~transformers.SchedulerType` for all possible
            values.
141
142
        warmup_ratio (:obj:`float`, `optional`, defaults to 0.0):
            Ratio of total training steps used for a linear warmup from 0 to :obj:`learning_rate`.
143
        warmup_steps (:obj:`int`, `optional`, defaults to 0):
144
145
            Number of steps used for a linear warmup from 0 to :obj:`learning_rate`. Overrides any effect of
            :obj:`warmup_ratio`.
146
        logging_dir (:obj:`str`, `optional`):
147
148
            `TensorBoard <https://www.tensorflow.org/tensorboard>`__ log directory. Will default to
            `runs/**CURRENT_DATETIME_HOSTNAME**`.
149
        logging_strategy (:obj:`str` or :class:`~transformers.trainer_utils.IntervalStrategy`, `optional`, defaults to :obj:`"steps"`):
150
151
152
153
154
155
            The logging strategy to adopt during training. Possible values are:

                * :obj:`"no"`: No logging is done during training.
                * :obj:`"epoch"`: Logging is done at the end of each epoch.
                * :obj:`"steps"`: Logging is done every :obj:`logging_steps`.

156
        logging_first_step (:obj:`bool`, `optional`, defaults to :obj:`False`):
Tiger's avatar
Tiger committed
157
            Whether to log and evaluate the first :obj:`global_step` or not.
158
        logging_steps (:obj:`int`, `optional`, defaults to 500):
159
            Number of update steps between two logs if :obj:`logging_strategy="steps"`.
160
161
162
163
164
165
166
        save_strategy (:obj:`str` or :class:`~transformers.trainer_utils.IntervalStrategy`, `optional`, defaults to :obj:`"steps"`):
            The checkpoint save strategy to adopt during training. Possible values are:

                * :obj:`"no"`: No save is done during training.
                * :obj:`"epoch"`: Save is done at the end of each epoch.
                * :obj:`"steps"`: Save is done every :obj:`save_steps`.

167
        save_steps (:obj:`int`, `optional`, defaults to 500):
168
            Number of updates steps before two checkpoint saves if :obj:`save_strategy="steps"`.
169
170
171
172
        save_total_limit (:obj:`int`, `optional`):
            If a value is passed, will limit the total amount of checkpoints. Deletes the older checkpoints in
            :obj:`output_dir`.
        no_cuda (:obj:`bool`, `optional`, defaults to :obj:`False`):
Alan deLevie's avatar
Alan deLevie committed
173
            Whether to not use CUDA even when it is available or not.
174
        seed (:obj:`int`, `optional`, defaults to 42):
175
176
177
            Random seed that will be set at the beginning of training. To ensure reproducibility across runs, use the
            :func:`~transformers.Trainer.model_init` function to instantiate the model if it has some randomly
            initialized parameters.
178
        fp16 (:obj:`bool`, `optional`, defaults to :obj:`False`):
179
            Whether to use 16-bit (mixed) precision training instead of 32-bit training.
180
        fp16_opt_level (:obj:`str`, `optional`, defaults to 'O1'):
181
182
            For :obj:`fp16` training, Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']. See details
            on the `Apex documentation <https://nvidia.github.io/apex/amp.html>`__.
183
184
185
186
        fp16_backend (:obj:`str`, `optional`, defaults to :obj:`"auto"`):
            The backend to use for mixed precision training. Must be one of :obj:`"auto"`, :obj:`"amp"` or
            :obj:`"apex"`. :obj:`"auto"` will use AMP or APEX depending on the PyTorch version detected, while the
            other choices will force the requested backend.
187
188
189
        fp16_full_eval (:obj:`bool`, `optional`, defaults to :obj:`False`):
            Whether to use full 16-bit precision evaluation instead of 32-bit. This will be faster and save memory but
            can harm metric values.
190
        local_rank (:obj:`int`, `optional`, defaults to -1):
191
            Rank of the process during distributed training.
192
        tpu_num_cores (:obj:`int`, `optional`):
Tiger's avatar
Tiger committed
193
            When training on TPU, the number of TPU cores (automatically passed by launcher script).
194
        debug (:obj:`bool`, `optional`, defaults to :obj:`False`):
195
196
197
198
            When training on TPU, whether to print debug metrics or not.
        dataloader_drop_last (:obj:`bool`, `optional`, defaults to :obj:`False`):
            Whether to drop the last incomplete batch (if the length of the dataset is not divisible by the batch size)
            or not.
199
200
201
        eval_steps (:obj:`int`, `optional`):
            Number of update steps between two evaluations if :obj:`evaluation_strategy="steps"`. Will default to the
            same value as :obj:`logging_steps` if not set.
Chady Kamar's avatar
Chady Kamar committed
202
        dataloader_num_workers (:obj:`int`, `optional`, defaults to 0):
Sylvain Gugger's avatar
Sylvain Gugger committed
203
204
            Number of subprocesses to use for data loading (PyTorch only). 0 means that the data will be loaded in the
            main process.
205
206
207
208
209
        past_index (:obj:`int`, `optional`, defaults to -1):
            Some models like :doc:`TransformerXL <../model_doc/transformerxl>` or :doc`XLNet <../model_doc/xlnet>` can
            make use of the past hidden states for their predictions. If this argument is set to a positive int, the
            ``Trainer`` will use the corresponding output (usually index 2) as the past state and feed it to the model
            at the next training step under the keyword argument ``mems``.
210
        run_name (:obj:`str`, `optional`):
211
            A descriptor for the run. Typically used for `wandb <https://www.wandb.com/>`_ logging.
212
        disable_tqdm (:obj:`bool`, `optional`):
213
214
215
            Whether or not to disable the tqdm progress bars and table of metrics produced by
            :class:`~transformers.notebook.NotebookTrainingTracker` in Jupyter Notebooks. Will default to :obj:`True`
            if the logging level is set to warn or lower (default), :obj:`False` otherwise.
216
        remove_unused_columns (:obj:`bool`, `optional`, defaults to :obj:`True`):
217
218
            If using :obj:`datasets.Dataset` datasets, whether or not to automatically remove the columns unused by the
            model forward method.
219

Sylvain Gugger's avatar
Sylvain Gugger committed
220
221
222
            (Note that this behavior is not implemented for :class:`~transformers.TFTrainer` yet.)
        label_names (:obj:`List[str]`, `optional`):
            The list of keys in your dictionary of inputs that correspond to the labels.
Sylvain Gugger's avatar
Sylvain Gugger committed
223
224

            Will eventually default to :obj:`["labels"]` except if the model used is one of the
Sylvain Gugger's avatar
Sylvain Gugger committed
225
            :obj:`XxxForQuestionAnswering` in which case it will default to :obj:`["start_positions",
Sylvain Gugger's avatar
Sylvain Gugger committed
226
227
228
            "end_positions"]`.
        load_best_model_at_end (:obj:`bool`, `optional`, defaults to :obj:`False`):
            Whether or not to load the best model found during training at the end of training.
229
230
231

            .. note::

232
233
                When set to :obj:`True`, the parameters :obj:`save_strategy` and :obj:`save_steps` will be ignored and
                the model will be saved after each evaluation.
Sylvain Gugger's avatar
Sylvain Gugger committed
234
        metric_for_best_model (:obj:`str`, `optional`):
235
236
237
238
239
            Use in conjunction with :obj:`load_best_model_at_end` to specify the metric to use to compare two different
            models. Must be the name of a metric returned by the evaluation with or without the prefix :obj:`"eval_"`.
            Will default to :obj:`"loss"` if unspecified and :obj:`load_best_model_at_end=True` (to use the evaluation
            loss).

Tiger's avatar
Tiger committed
240
            If you set this value, :obj:`greater_is_better` will default to :obj:`True`. Don't forget to set it to
Sylvain Gugger's avatar
Sylvain Gugger committed
241
242
243
244
            :obj:`False` if your metric is better when lower.
        greater_is_better (:obj:`bool`, `optional`):
            Use in conjunction with :obj:`load_best_model_at_end` and :obj:`metric_for_best_model` to specify if better
            models should have a greater metric or not. Will default to:
245
246
247
248

            - :obj:`True` if :obj:`metric_for_best_model` is set to a value that isn't :obj:`"loss"` or
              :obj:`"eval_loss"`.
            - :obj:`False` if :obj:`metric_for_best_model` is not set, or set to :obj:`"loss"` or :obj:`"eval_loss"`.
249
        ignore_data_skip (:obj:`bool`, `optional`, defaults to :obj:`False`):
250
251
252
            When resuming training, whether or not to skip the epochs and batches to get the data loading at the same
            stage as in the previous training. If set to :obj:`True`, the training will begin faster (as that skipping
            step can take a long time) but will not yield the same results as the interrupted training would have.
253
        sharded_ddp (:obj:`bool`, :obj:`str` or list of :class:`~transformers.trainer_utils.ShardedDDPOption`, `optional`, defaults to :obj:`False`):
254
255
            Use Sharded DDP training from `FairScale <https://github.com/facebookresearch/fairscale>`__ (in distributed
            training only). This is an experimental feature.
256
257
258
259
260
261
262
263
264
265
266
267
268

            A list of options along the following:

            - :obj:`"simple"`: to use first instance of sharded DDP released by fairscale (:obj:`ShardedDDP`) similar
              to ZeRO-2.
            - :obj:`"zero_dp_2"`: to use the second instance of sharded DPP released by fairscale
              (:obj:`FullyShardedDDP`) in Zero-2 mode (with :obj:`reshard_after_forward=False`).
            - :obj:`"zero_dp_3"`: to use the second instance of sharded DPP released by fairscale
              (:obj:`FullyShardedDDP`) in Zero-3 mode (with :obj:`reshard_after_forward=True`).
            - :obj:`"offload"`: to add ZeRO-offload (only compatible with :obj:`"zero_dp_2"` and :obj:`"zero_dp_3"`).

            If a string is passed, it will be split on space. If a bool is passed, it will be converted to an empty
            list for :obj:`False` and :obj:`["simple"]` for :obj:`True`.
269
        deepspeed (:obj:`str` or :obj:`dict`, `optional`):
270
            Use `Deepspeed <https://github.com/microsoft/deepspeed>`__. This is an experimental feature and its API may
271
272
            evolve in the future. The value is either the location of DeepSpeed json config file (e.g.,
            ``ds_config.json``) or an already loaded json file as a :obj:`dict`"
Sylvain Gugger's avatar
Sylvain Gugger committed
273
274
275
276
277
278
279
        label_smoothing_factor (:obj:`float`, `optional`, defaults to 0.0):
            The label smoothing factor to use. Zero means no label smoothing, otherwise the underlying onehot-encoded
            labels are changed from 0s and 1s to :obj:`label_smoothing_factor/num_labels` and :obj:`1 -
            label_smoothing_factor + label_smoothing_factor/num_labels` respectively.
        adafactor (:obj:`bool`, `optional`, defaults to :obj:`False`):
            Whether or not to use the :class:`~transformers.Adafactor` optimizer instead of
            :class:`~transformers.AdamW`.
280
        group_by_length (:obj:`bool`, `optional`, defaults to :obj:`False`):
JohnnyC08's avatar
JohnnyC08 committed
281
            Whether or not to group together samples of roughly the same length in the training dataset (to minimize
282
            padding applied and be more efficient). Only useful if applying dynamic padding.
283
284
285
286
        length_column_name (:obj:`str`, `optional`, defaults to :obj:`"length"`):
            Column name for precomputed lengths. If the column exists, grouping by length will use these values rather
            than computing them on train startup. Ignored unless :obj:`group_by_length` is :obj:`True` and the dataset
            is an instance of :obj:`Dataset`.
287
        report_to (:obj:`str` or :obj:`List[str]`, `optional`, defaults to :obj:`"all"`):
288
            The list of integrations to report the results and logs to. Supported platforms are :obj:`"azure_ml"`,
289
290
            :obj:`"comet_ml"`, :obj:`"mlflow"`, :obj:`"tensorboard"` and :obj:`"wandb"`. Use :obj:`"all"` to report to
            all integrations installed, :obj:`"none"` for no integrations.
291
292
        ddp_find_unused_parameters (:obj:`bool`, `optional`):
            When using distributed training, the value of the flag :obj:`find_unused_parameters` passed to
293
            :obj:`DistributedDataParallel`. Will default to :obj:`False` if gradient checkpointing is used, :obj:`True`
294
            otherwise.
Sylvain Gugger's avatar
Sylvain Gugger committed
295
        dataloader_pin_memory (:obj:`bool`, `optional`, defaults to :obj:`True`):
296
            Whether you want to pin memory in data loaders or not. Will default to :obj:`True`.
Sylvain Gugger's avatar
Sylvain Gugger committed
297
        skip_memory_metrics (:obj:`bool`, `optional`, defaults to :obj:`False`):
298
            Whether to skip adding of memory profiler reports to metrics. Defaults to :obj:`False`.
Sylvain Gugger's avatar
Sylvain Gugger committed
299
300
301
302
303
        push_to_hub (:obj:`bool`, `optional`, defaults to :obj:`False`):
            Whether or not to upload the trained model to the hub after training. This argument is not directly used by
            :class:`~transformers.Trainer`, it's intended to be used by your training/evaluation scripts instead. See
            the `example scripts <https://github.com/huggingface/transformers/tree/master/examples>`__ for more
            details.
304
305
306
307
308
        resume_from_checkpoint (:obj:`str`, `optional`):
            The path to a folder with a valid checkpoint for your model. This argument is not directly used by
            :class:`~transformers.Trainer`, it's intended to be used by your training/evaluation scripts instead. See
            the `example scripts <https://github.com/huggingface/transformers/tree/master/examples>`__ for more
            details.
309
310
    """

311
    output_dir: str = field(
312
        metadata={"help": "The output directory where the model predictions and checkpoints will be written."},
313
314
    )
    overwrite_output_dir: bool = field(
315
316
317
318
319
320
321
        default=False,
        metadata={
            "help": (
                "Overwrite the content of the output directory."
                "Use this to continue training if output_dir points to a checkpoint directory."
            )
        },
322
323
324
    )

    do_train: bool = field(default=False, metadata={"help": "Whether to run training."})
325
    do_eval: bool = field(default=False, metadata={"help": "Whether to run eval on the dev set."})
Julien Chaumond's avatar
Julien Chaumond committed
326
    do_predict: bool = field(default=False, metadata={"help": "Whether to run predictions on the test set."})
327
    evaluation_strategy: IntervalStrategy = field(
328
        default="no",
Sylvain Gugger's avatar
Sylvain Gugger committed
329
        metadata={"help": "The evaluation strategy to use."},
330
    )
331
    prediction_loss_only: bool = field(
Lysandre's avatar
Lysandre committed
332
333
        default=False,
        metadata={"help": "When performing evaluation and predictions, only returns the loss."},
334
    )
335

336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
    per_device_train_batch_size: int = field(
        default=8, metadata={"help": "Batch size per GPU/TPU core/CPU for training."}
    )
    per_device_eval_batch_size: int = field(
        default=8, metadata={"help": "Batch size per GPU/TPU core/CPU for evaluation."}
    )

    per_gpu_train_batch_size: Optional[int] = field(
        default=None,
        metadata={
            "help": "Deprecated, the use of `--per_device_train_batch_size` is preferred. "
            "Batch size per GPU/TPU core/CPU for training."
        },
    )
    per_gpu_eval_batch_size: Optional[int] = field(
        default=None,
        metadata={
            "help": "Deprecated, the use of `--per_device_eval_batch_size` is preferred."
            "Batch size per GPU/TPU core/CPU for evaluation."
        },
    )

358
    gradient_accumulation_steps: int = field(
359
360
        default=1,
        metadata={"help": "Number of updates steps to accumulate before performing a backward/update pass."},
361
    )
362
363
364
365
    eval_accumulation_steps: Optional[int] = field(
        default=None,
        metadata={"help": "Number of predictions steps to accumulate before moving the tensors to the CPU."},
    )
366

367
368
369
370
371
    learning_rate: float = field(default=5e-5, metadata={"help": "The initial learning rate for AdamW."})
    weight_decay: float = field(default=0.0, metadata={"help": "Weight decay for AdamW if we apply some."})
    adam_beta1: float = field(default=0.9, metadata={"help": "Beta1 for AdamW optimizer"})
    adam_beta2: float = field(default=0.999, metadata={"help": "Beta2 for AdamW optimizer"})
    adam_epsilon: float = field(default=1e-8, metadata={"help": "Epsilon for AdamW optimizer."})
372
373
374
375
376
377
378
    max_grad_norm: float = field(default=1.0, metadata={"help": "Max gradient norm."})

    num_train_epochs: float = field(default=3.0, metadata={"help": "Total number of training epochs to perform."})
    max_steps: int = field(
        default=-1,
        metadata={"help": "If > 0: set total number of training steps to perform. Override num_train_epochs."},
    )
Sylvain Gugger's avatar
Sylvain Gugger committed
379
380
381
382
    lr_scheduler_type: SchedulerType = field(
        default="linear",
        metadata={"help": "The scheduler type to use."},
    )
383
384
385
    warmup_ratio: float = field(
        default=0.0, metadata={"help": "Linear warmup over warmup_ratio fraction of total steps."}
    )
386
387
    warmup_steps: int = field(default=0, metadata={"help": "Linear warmup over warmup_steps."})

Julien Plu's avatar
Julien Plu committed
388
    logging_dir: Optional[str] = field(default_factory=default_logdir, metadata={"help": "Tensorboard log dir."})
389
    logging_strategy: IntervalStrategy = field(
390
391
392
        default="steps",
        metadata={"help": "The logging strategy to use."},
    )
393
    logging_first_step: bool = field(default=False, metadata={"help": "Log the first global_step"})
394
    logging_steps: int = field(default=500, metadata={"help": "Log every X updates steps."})
395
396
397
398
    save_strategy: IntervalStrategy = field(
        default="steps",
        metadata={"help": "The checkpoint save strategy to use."},
    )
399
400
401
402
    save_steps: int = field(default=500, metadata={"help": "Save checkpoint every X updates steps."})
    save_total_limit: Optional[int] = field(
        default=None,
        metadata={
403
404
405
406
            "help": (
                "Limit the total amount of checkpoints."
                "Deletes the older checkpoints in the output_dir. Default is unlimited checkpoints"
            )
407
408
        },
    )
Lysandre Debut's avatar
Lysandre Debut committed
409
    no_cuda: bool = field(default=False, metadata={"help": "Do not use CUDA even when it is available"})
410
    seed: int = field(default=42, metadata={"help": "Random seed that will be set at the beginning of training."})
411
412
413

    fp16: bool = field(
        default=False,
414
        metadata={"help": "Whether to use 16-bit (mixed) precision instead of 32-bit"},
415
416
417
418
    )
    fp16_opt_level: str = field(
        default="O1",
        metadata={
419
420
421
422
            "help": (
                "For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
                "See details at https://nvidia.github.io/apex/amp.html"
            )
423
424
        },
    )
425
426
427
428
    fp16_backend: str = field(
        default="auto",
        metadata={"help": "The backend to be used for mixed precision.", "choices": ["auto", "amp", "apex"]},
    )
429
430
431
432
    fp16_full_eval: bool = field(
        default=False,
        metadata={"help": "Whether to use full 16-bit precision evaluation instead of 32-bit"},
    )
433
    local_rank: int = field(default=-1, metadata={"help": "For distributed training: local_rank"})
Julien Chaumond's avatar
Julien Chaumond committed
434

Lysandre Debut's avatar
Lysandre Debut committed
435
436
437
    tpu_num_cores: Optional[int] = field(
        default=None, metadata={"help": "TPU: Number of TPU cores (automatically passed by launcher script)"}
    )
438
439
440
441
442
    tpu_metrics_debug: bool = field(
        default=False,
        metadata={"help": "Deprecated, the use of `--debug` is preferred. TPU: Whether to print debug metrics"},
    )
    debug: bool = field(default=False, metadata={"help": "Whether to print debug metrics on TPU"})
Lysandre Debut's avatar
Lysandre Debut committed
443

Setu Shah's avatar
Setu Shah committed
444
445
446
    dataloader_drop_last: bool = field(
        default=False, metadata={"help": "Drop the last incomplete batch if it is not divisible by the batch size."}
    )
447
    eval_steps: int = field(default=None, metadata={"help": "Run an evaluation every X steps."})
Chady Kamar's avatar
Chady Kamar committed
448
449
    dataloader_num_workers: int = field(
        default=0,
Sylvain Gugger's avatar
Sylvain Gugger committed
450
451
452
        metadata={
            "help": "Number of subprocesses to use for data loading (PyTorch only). 0 means that the data will be loaded in the main process."
        },
Chady Kamar's avatar
Chady Kamar committed
453
    )
Setu Shah's avatar
Setu Shah committed
454

455
456
457
458
459
    past_index: int = field(
        default=-1,
        metadata={"help": "If >=0, uses the corresponding part of the output as the past state for next step."},
    )

460
461
462
    run_name: Optional[str] = field(
        default=None, metadata={"help": "An optional descriptor for the run. Notably used for wandb logging."}
    )
463
464
465
466
    disable_tqdm: Optional[bool] = field(
        default=None, metadata={"help": "Whether or not to disable the tqdm progress bars."}
    )

467
468
469
    remove_unused_columns: Optional[bool] = field(
        default=True, metadata={"help": "Remove columns not required by the model when using an nlp.Dataset."}
    )
Sylvain Gugger's avatar
Sylvain Gugger committed
470
471
472
473
    label_names: Optional[List[str]] = field(
        default=None, metadata={"help": "The list of keys in your dictionary of inputs that correspond to the labels."}
    )

474
475
476
477
478
479
480
481
482
483
    load_best_model_at_end: Optional[bool] = field(
        default=False,
        metadata={"help": "Whether or not to load the best model found during training at the end of training."},
    )
    metric_for_best_model: Optional[str] = field(
        default=None, metadata={"help": "The metric to use to compare two different models."}
    )
    greater_is_better: Optional[bool] = field(
        default=None, metadata={"help": "Whether the `metric_for_best_model` should be maximized or not."}
    )
484
485
486
487
488
489
    ignore_data_skip: bool = field(
        default=False,
        metadata={
            "help": "When resuming training, whether or not to skip the first epochs and batches to get to the same training data."
        },
    )
490
491
492
493
494
    sharded_ddp: str = field(
        default="",
        metadata={
            "help": "Whether or not to use sharded DDP training (in distributed training only). The base option "
            "should be `simple`, `zero_dp_2` or `zero_dp_3` and you can add CPU-offload to `zero_dp_2` or `zero_dp_3` "
495
496
            "like this: zero_dp_2 offload` or `zero_dp_3 offload`. You can add auto-wrap to `zero_dp_2` or "
            "with the same syntax: zero_dp_2 auto_wrap` or `zero_dp_3 auto_wrap`.",
497
        },
498
    )
499
500
    deepspeed: Optional[str] = field(
        default=None,
501
502
503
        metadata={
            "help": "Enable deepspeed and pass the path to deepspeed json config file (e.g. ds_config.json) or an already loaded json file as a dict"
        },
504
    )
Sylvain Gugger's avatar
Sylvain Gugger committed
505
506
507
    label_smoothing_factor: float = field(
        default=0.0, metadata={"help": "The label smoothing epsilon to apply (zero means no label smoothing)."}
    )
508
    adafactor: bool = field(default=False, metadata={"help": "Whether or not to replace AdamW by Adafactor."})
509
510
511
512
    group_by_length: bool = field(
        default=False,
        metadata={"help": "Whether or not to group samples of roughly the same length together when batching."},
    )
513
514
515
516
    length_column_name: Optional[str] = field(
        default="length",
        metadata={"help": "Column name with precomputed lengths to use when grouping by length."},
    )
517
518
519
    report_to: Optional[List[str]] = field(
        default=None, metadata={"help": "The list of integrations to report the results and logs to."}
    )
520
521
522
523
524
525
526
    ddp_find_unused_parameters: Optional[bool] = field(
        default=None,
        metadata={
            "help": "When using distributed training, the value of the flag `find_unused_parameters` passed to "
            "`DistributedDataParallel`."
        },
    )
527
528
529
    dataloader_pin_memory: bool = field(
        default=True, metadata={"help": "Whether or not to pin memory for DataLoader."}
    )
530
531
532
    skip_memory_metrics: bool = field(
        default=False, metadata={"help": "Whether or not to skip adding of memory profiler reports to metrics."}
    )
533
534
535
    use_legacy_prediction_loop: bool = field(
        default=False, metadata={"help": "Whether or not to use the legacy prediction_loop in the Trainer."}
    )
Sylvain Gugger's avatar
Sylvain Gugger committed
536
537
538
    push_to_hub: bool = field(
        default=False, metadata={"help": "Whether or not to upload the trained model to the model hub after training."}
    )
539
540
541
542
    resume_from_checkpoint: Optional[str] = field(
        default=None,
        metadata={"help": "The path to a folder with a valid checkpoint for your model."},
    )
543
    _n_gpu: int = field(init=False, repr=False, default=-1)
Sylvain Gugger's avatar
Sylvain Gugger committed
544
545
546
547
    mp_parameters: str = field(
        default="",
        metadata={"help": "Used by the SageMaker launcher to send mp-specific args. Ignored in Trainer"},
    )
548

Sylvain Gugger's avatar
Sylvain Gugger committed
549
    def __post_init__(self):
550
551
552
553
554
555
        # Handle --use_env option in torch.distributed.launch (local_rank not passed as an arg then).
        # This needs to happen before any call to self.device or self.n_gpu.
        env_local_rank = int(os.environ.get("LOCAL_RANK", -1))
        if env_local_rank != -1 and env_local_rank != self.local_rank:
            self.local_rank = env_local_rank

556
557
558
559
560
561
562
563
        # expand paths, if not os.makedirs("~/bar") will make directory
        # in the current directory instead of the actual home
        # 聽see https://github.com/huggingface/transformers/issues/10628
        if self.output_dir is not None:
            self.output_dir = os.path.expanduser(self.output_dir)
        if self.logging_dir is not None:
            self.logging_dir = os.path.expanduser(self.logging_dir)

Sylvain Gugger's avatar
Sylvain Gugger committed
564
565
        if self.disable_tqdm is None:
            self.disable_tqdm = logger.getEffectiveLevel() > logging.WARN
566
567
568
569
570
571

        if isinstance(self.evaluation_strategy, EvaluationStrategy):
            warnings.warn(
                "using `EvaluationStrategy` for `evaluation_strategy` is deprecated and will be removed in version 5 of 馃 Transformers. Use `IntervalStrategy` instead",
                FutureWarning,
            )
572
573
            # Go back to the underlying string or we won't be able to instantiate `IntervalStrategy` on it.
            self.evaluation_strategy = self.evaluation_strategy.value
574
575
576
577
578

        self.evaluation_strategy = IntervalStrategy(self.evaluation_strategy)
        self.logging_strategy = IntervalStrategy(self.logging_strategy)
        self.save_strategy = IntervalStrategy(self.save_strategy)

Sylvain Gugger's avatar
Sylvain Gugger committed
579
        self.lr_scheduler_type = SchedulerType(self.lr_scheduler_type)
580
        if self.do_eval is False and self.evaluation_strategy != IntervalStrategy.NO:
581
            self.do_eval = True
582
583
        if self.eval_steps is None:
            self.eval_steps = self.logging_steps
584

585
586
587
588
        if self.load_best_model_at_end and self.metric_for_best_model is None:
            self.metric_for_best_model = "loss"
        if self.greater_is_better is None and self.metric_for_best_model is not None:
            self.greater_is_better = self.metric_for_best_model not in ["loss", "eval_loss"]
589
590
        if self.run_name is None:
            self.run_name = self.output_dir
591

592
593
594
595
        if is_torch_available() and self.device.type != "cuda" and (self.fp16 or self.fp16_full_eval):
            raise ValueError(
                "Mixed precision training with AMP or APEX (`--fp16`) and FP16 evaluation can only be used on CUDA devices."
            )
596
        if self.report_to is None:
597
598
599
600
601
602
603
            logger.info(
                "The default value for the training argument `--report_to` will change in v5 (from all installed "
                "integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as "
                "now. You should start updating your code and make this info disappear :-)."
            )
            self.report_to = "all"
        if self.report_to == "all" or self.report_to == ["all"]:
604
605
606
607
            # Import at runtime to avoid a circular import.
            from .integrations import get_available_reporting_integrations

            self.report_to = get_available_reporting_integrations()
608
609
610
611
        elif self.report_to == "none" or self.report_to == ["none"]:
            self.report_to = []
        elif not isinstance(self.report_to, list):
            self.report_to = [self.report_to]
612

613
614
615
616
617
618
619
        if self.warmup_ratio < 0 or self.warmup_ratio > 1:
            raise ValueError("warmup_ratio must lie in range [0,1]")
        elif self.warmup_ratio > 0 and self.warmup_steps > 0:
            logger.info(
                "Both warmup_ratio and warmup_steps given, warmup_steps will override any effect of warmup_ratio during training"
            )

620
621
622
623
624
625
626
627
628
        if isinstance(self.sharded_ddp, bool):
            self.sharded_ddp = "simple" if self.sharded_ddp else ""
        if isinstance(self.sharded_ddp, str):
            self.sharded_ddp = [ShardedDDPOption(s) for s in self.sharded_ddp.split()]
        if self.sharded_ddp == [ShardedDDPOption.OFFLOAD]:
            raise ValueError(
                "`--sharded_ddp offload` can't work on its own. It needs to be added to `--sharded_ddp zero_dp_2` or "
                '`--sharded_ddp zero_dp_3`. For example, `--sharded_ddp "zero_dp_2 offload"`.'
            )
629
        elif len(self.sharded_ddp) > 1 and ShardedDDPOption.SIMPLE in self.sharded_ddp:
630
631
632
633
            raise ValueError("`--sharded_ddp simple` is not compatible with any other option.")
        elif ShardedDDPOption.ZERO_DP_2 in self.sharded_ddp and ShardedDDPOption.ZERO_DP_3 in self.sharded_ddp:
            raise ValueError("`--sharded_ddp zero_dp_2` is not compatible with `--sharded_ddp zero_dp_3`.")

634
635
636
637
638
639
640
641
        if self.deepspeed:
            # - must be run very last in arg parsing, since it will use a lot of these settings.
            # - must be run before the model is created.
            from transformers.integrations import DeepSpeedConfigHF

            # will be used later by the Trainer (leave self.deepspeed unmodified in case a user relies on it not to be modified)
            self.deepspeed_config_hf = DeepSpeedConfigHF(self)

642
643
644
645
646
647
648
649
    def __repr__(self):
        # We override the default repr to remove deprecated arguments from the repr. This method should be removed once
        # those deprecated arguments are removed form TrainingArguments. (TODO: v5)
        self_as_dict = asdict(self)
        del self_as_dict["per_gpu_train_batch_size"]
        del self_as_dict["per_gpu_eval_batch_size"]
        attrs_as_str = [f"{k}={v}" for k, v in self_as_dict.items()]
        return f"{self.__class__.__name__}({', '.join(attrs_as_str)})"
650

Julien Chaumond's avatar
Julien Chaumond committed
651
652
    @property
    def train_batch_size(self) -> int:
653
654
655
        """
        The actual batch size for training (may differ from :obj:`per_gpu_train_batch_size` in distributed training).
        """
656
657
658
659
660
661
        if self.per_gpu_train_batch_size:
            logger.warning(
                "Using deprecated `--per_gpu_train_batch_size` argument which will be removed in a future "
                "version. Using `--per_device_train_batch_size` is preferred."
            )
        per_device_batch_size = self.per_gpu_train_batch_size or self.per_device_train_batch_size
662
        train_batch_size = per_device_batch_size * max(1, self.n_gpu)
663
        return train_batch_size
Julien Chaumond's avatar
Julien Chaumond committed
664
665
666

    @property
    def eval_batch_size(self) -> int:
667
668
669
        """
        The actual batch size for evaluation (may differ from :obj:`per_gpu_eval_batch_size` in distributed training).
        """
670
671
672
673
674
675
        if self.per_gpu_eval_batch_size:
            logger.warning(
                "Using deprecated `--per_gpu_eval_batch_size` argument which will be removed in a future "
                "version. Using `--per_device_eval_batch_size` is preferred."
            )
        per_device_batch_size = self.per_gpu_eval_batch_size or self.per_device_eval_batch_size
676
        eval_batch_size = per_device_batch_size * max(1, self.n_gpu)
677
        return eval_batch_size
Julien Chaumond's avatar
Julien Chaumond committed
678
679
680

    @cached_property
    @torch_required
681
    def _setup_devices(self) -> "torch.device":
Julien Chaumond's avatar
Julien Chaumond committed
682
683
684
        logger.info("PyTorch: setting up devices")
        if self.no_cuda:
            device = torch.device("cpu")
685
            self._n_gpu = 0
686
        elif is_torch_tpu_available():
Lysandre Debut's avatar
Lysandre Debut committed
687
            device = xm.xla_device()
688
            self._n_gpu = 0
Sylvain Gugger's avatar
Sylvain Gugger committed
689
690
691
692
693
        elif is_sagemaker_mp_enabled():
            local_rank = smp.local_rank()
            device = torch.device("cuda", local_rank)
            self._n_gpu = 1
        elif is_sagemaker_dp_enabled():
694
695
            sm_dist.init_process_group()
            self.local_rank = sm_dist.get_local_rank()
Sylvain Gugger's avatar
Sylvain Gugger committed
696
697
            device = torch.device("cuda", self.local_rank)
            self._n_gpu = 1
698
699
700
701
702
703
704
705
706
707
708
709
        elif self.deepspeed:
            # deepspeed performs its own DDP internally, and requires the program to be started with:
            # deepspeed  ./program.py
            # rather than:
            # python -m torch.distributed.launch --nproc_per_node=2 ./program.py
            from .integrations import is_deepspeed_available

            if not is_deepspeed_available():
                raise ImportError("--deepspeed requires deepspeed: `pip install deepspeed`.")
            import deepspeed

            deepspeed.init_distributed()
710
711
712
713
714
715

            # workaround for setups like notebooks where the launcher can't be used,
            # but deepspeed requires a dist env.
            # env LOCAL_RANK could be set manually by the user, or via init_distributed if mpi4py is installed
            self.local_rank = int(os.environ.get("LOCAL_RANK", "-1"))

716
717
            device = torch.device("cuda", self.local_rank)
            self._n_gpu = 1
Julien Chaumond's avatar
Julien Chaumond committed
718
719
720
        elif self.local_rank == -1:
            # if n_gpu is > 1 we'll use nn.DataParallel.
            # If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
721
722
723
724
725
            # Explicitly set CUDA to the first (index 0) CUDA device, otherwise `set_device` will
            # trigger an error that a device index is missing. Index 0 takes into account the
            # GPUs available in the environment, so `CUDA_VISIBLE_DEVICES=1,2` with `cuda:0`
            # will use the first GPU in that env, i.e. GPU#1
            device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
726
727
            # Sometimes the line in the postinit has not been run before we end up here, so just checking we're not at
            # the default value.
728
            self._n_gpu = torch.cuda.device_count()
Julien Chaumond's avatar
Julien Chaumond committed
729
730
        else:
            # Here, we'll use torch.distributed.
731
            # Initializes the distributed backend which will take care of synchronizing nodes/GPUs
732
            torch.distributed.init_process_group(backend="nccl")
Julien Chaumond's avatar
Julien Chaumond committed
733
            device = torch.device("cuda", self.local_rank)
734
            self._n_gpu = 1
735
736
737
738

        if device.type == "cuda":
            torch.cuda.set_device(device)

739
        return device
Julien Chaumond's avatar
Julien Chaumond committed
740
741
742
743

    @property
    @torch_required
    def device(self) -> "torch.device":
744
745
746
        """
        The device used by this process.
        """
747
        return self._setup_devices
Julien Chaumond's avatar
Julien Chaumond committed
748
749
750
751

    @property
    @torch_required
    def n_gpu(self):
752
753
754
755
756
757
758
        """
        The number of GPUs used by this process.

        Note:
            This will only be greater than one when you have multiple GPUs available but are not using distributed
            training. For distributed training, it will always be 1.
        """
759
760
761
        # Make sure `self._n_gpu` is properly setup.
        _ = self._setup_devices
        return self._n_gpu
Julien Chaumond's avatar
Julien Chaumond committed
762

763
764
765
766
767
768
769
770
    @property
    @torch_required
    def parallel_mode(self):
        """
        The current mode used for parallelism if multiple GPUs/TPU cores are available. One of:

        - :obj:`ParallelMode.NOT_PARALLEL`: no parallelism (CPU or one GPU).
        - :obj:`ParallelMode.NOT_DISTRIBUTED`: several GPUs in one single process (uses :obj:`torch.nn.DataParallel`).
771
        - :obj:`ParallelMode.DISTRIBUTED`: several GPUs, each having its own process (uses
772
773
774
775
776
          :obj:`torch.nn.DistributedDataParallel`).
        - :obj:`ParallelMode.TPU`: several TPU cores.
        """
        if is_torch_tpu_available():
            return ParallelMode.TPU
Sylvain Gugger's avatar
Sylvain Gugger committed
777
778
779
780
        elif is_sagemaker_mp_enabled():
            return ParallelMode.SAGEMAKER_MODEL_PARALLEL
        elif is_sagemaker_dp_enabled():
            return ParallelMode.SAGEMAKER_DATA_PARALLEL
781
782
783
784
785
786
787
        elif self.local_rank != -1:
            return ParallelMode.DISTRIBUTED
        elif self.n_gpu > 1:
            return ParallelMode.NOT_DISTRIBUTED
        else:
            return ParallelMode.NOT_PARALLEL

788
789
790
791
792
793
794
795
    @property
    @torch_required
    def world_size(self):
        """
        The number of processes used in parallel.
        """
        if is_torch_tpu_available():
            return xm.xrt_world_size()
Sylvain Gugger's avatar
Sylvain Gugger committed
796
797
798
        elif is_sagemaker_mp_enabled():
            return smp.dp_size()
        elif is_sagemaker_dp_enabled():
799
800
801
802
803
            return sm_dist.get_world_size()
        elif self.local_rank != -1:
            return torch.distributed.get_world_size()
        return 1

804
805
806
807
808
809
810
811
    @property
    @torch_required
    def process_index(self):
        """
        The number of processes used in parallel.
        """
        if is_torch_tpu_available():
            return xm.get_ordinal()
Sylvain Gugger's avatar
Sylvain Gugger committed
812
813
814
        elif is_sagemaker_mp_enabled():
            return smp.dp_rank()
        elif is_sagemaker_dp_enabled():
815
816
817
818
819
            return sm_dist.get_rank()
        elif self.local_rank != -1:
            return torch.distributed.get_rank()
        return 0

820
821
822
823
824
    @property
    def place_model_on_device(self):
        """
        Can be subclassed and overridden for some specific integrations.
        """
Sylvain Gugger's avatar
Sylvain Gugger committed
825
        return not is_sagemaker_mp_enabled()
826

Sylvain Gugger's avatar
Sylvain Gugger committed
827
828
829
830
831
    @property
    def _no_sync_in_gradient_accumulation(self):
        """
        Whether or not to use no_sync for the gradients when doing gradient accumulation.
        """
832
        return not (self.deepspeed or is_sagemaker_dp_enabled() or is_sagemaker_mp_enabled())
Sylvain Gugger's avatar
Sylvain Gugger committed
833

834
835
836
837
    def to_dict(self):
        """
        Serializes this instance while replace `Enum` by their values (for JSON serialization support).
        """
838
        d = asdict(self)
839
840
841
        for k, v in d.items():
            if isinstance(v, Enum):
                d[k] = v.value
842
843
            if isinstance(v, list) and len(v) > 0 and isinstance(v[0], Enum):
                d[k] = [x.value for x in v]
844
845
        return d

Julien Chaumond's avatar
Julien Chaumond committed
846
847
848
849
    def to_json_string(self):
        """
        Serializes this instance to a JSON string.
        """
850
        return json.dumps(self.to_dict(), indent=2)
851
852
853
854
855

    def to_sanitized_dict(self) -> Dict[str, Any]:
        """
        Sanitized serialization to use with TensorBoard鈥檚 hparams
        """
856
        d = self.to_dict()
857
858
        d = {**d, **{"train_batch_size": self.train_batch_size, "eval_batch_size": self.eval_batch_size}}

859
860
861
        valid_types = [bool, int, float, str]
        if is_torch_available():
            valid_types.append(torch.Tensor)
862

863
        return {k: v if type(v) in valid_types else str(v) for k, v in d.items()}
864
865
866
867
868
869


class ParallelMode(Enum):
    NOT_PARALLEL = "not_parallel"
    NOT_DISTRIBUTED = "not_distributed"
    DISTRIBUTED = "distributed"
Sylvain Gugger's avatar
Sylvain Gugger committed
870
871
    SAGEMAKER_MODEL_PARALLEL = "sagemaker_model_parallel"
    SAGEMAKER_DATA_PARALLEL = "sagemaker_data_parallel"
872
    TPU = "tpu"