gpt2xl-flash.yaml

# @package _global_
defaults:
  - /experiment/owt/gpt2l-flash.yaml
  - override /model/gpt2model: gpt2-xlarge

# Can enable mlp_checkpoint_lvl to fit to A100 40GB
# model:
#   config:
#     # mlp_checkpoint_lvl: ${eval:"[1] * 18 + [2] * 18"}
#     mlp_checkpoint_lvl: 1

datamodule:
  batch_size: ${eval:"2 if ${train.gpu_mem} < 24 else (4 if ${train.gpu_mem} < 40 else 8)"}
  # With adamw-zero optimizer:
  # checkpoint_lvl=1, batch size = 4: mem 37GB, 4650ms / batch of 512 (285ms * 15 + 375ms * 1)
  # checkpoint_lvl=1, batch size = 8: mem 46GB, 4330ms / batch of 512 (530ms * 7 + 620ms * 1)
  # checkpoint_lvl=2, batch size = 8: mem 41GB, 4570ms / batch of 512 (560ms * 7 + 650ms * 1)
  # With adamw-apex-distributed optimizer:
  # checkpoint_lvl=1, batch size = 8: mem 41.5GB, 4500ms / batch of 512 (550ms * 7 + 650ms * 1)
  # checkpoint_lvl=1 for 24 layers and checkpoint_lvl=2 for 24 layers,
  # batch size = 8: mem 39GB, 4640ms / batch of 512 (565ms * 7 + 675ms * 1)