Skip to content

GitLab

Switch branch/tag
Find file Normal viewHistoryPermalink
gpt2l.yaml 320 Bytes
Newer Older
Tri Dao's avatar
Release training code  
Tri Dao committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14 
# @package _global_
defaults:
  - /experiment/owt/gpt2m.yaml
  - override /model/gpt2model: gpt2-large
  - override /optimizer: adamw-zero

datamodule:
  batch_size: 4  # Per GPU

trainer:
  strategy:
    _target_: src.utils.ddp_zero1.DDPStrategyZero1
    find_unused_parameters: False
    gradient_as_bucket_view: True