ENV_VARS:
  CUDA_DEVICE_MAX_CONNECTIONS: 1
  NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
  NCCL_ALGO: Tree
  CUBLAS_WORKSPACE_CONFIG: :4096:8
  GPUS_PER_NODE: 8
MODEL_ARGS:
  --num-layers: 12
  --hidden-size: 624
  --attention-dropout: 0.0
  --hidden-dropout: 0.0
  --num-attention-heads: 12
  --log-params-norm: true
  --log-num-zeros-in-grad: true
  --log-validation-ppl-to-tensorboard: true
  --log-timers-to-tensorboard: true
  --tensorboard-dir: ${TENSORBOARD_PATH}
  --micro-batch-size: 4
  --global-batch-size: 32
  --seq-length: 1024
  --max-position-embeddings: 1024
  --train-iters: 50
  --timing-log-level: 2
  --lr-decay-iters: 320000
  --save: ${CHECKPOINT_PATH}
  --load: ${CHECKPOINT_PATH}
  --split: 949,50,1
  --tokenizer-type: NullTokenizer
  --vocab-size: 8192
  --distributed-backend: nccl
  --lr: 0.00015
  --lr-decay-style: cosine
  --min-lr: 1.0e-5
  --weight-decay: 1e-2
  --clip-grad: 1.0
  --lr-warmup-fraction: .01
  --log-interval: 1
  --save-interval: 10000
  --eval-interval: 1000
  --eval-iters: 10
  --transformer-impl: transformer_engine
  --tensor-model-parallel-size: 4
  --pipeline-model-parallel-size: 1
  --encoder-pipeline-model-parallel-size: 1
  --encoder-tensor-model-parallel-size: 4
  --deterministic-mode: true
  --attention-softmax-in-fp32: true
  --ckpt-format: torch
  --no-gradient-accumulation-fusion: true
  --bf16: true
  --img-h: 336
  --img-w: 336
  --patch-dim: 14
  --mock-data: true
  --freeze-ViT: true
  --freeze-LM: true
  --use-distributed-optimizer: true
TEST_TYPE: regular
