ENV_VARS:
  CUDA_DEVICE_MAX_CONNECTIONS: 1
  NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
  NCCL_ALGO: Tree
  CUBLAS_WORKSPACE_CONFIG: :4096:8
MODEL_ARGS:
  --num-layers: 12
  --hidden-size: 512
  --num-attention-heads: 8
  --log-params-norm: true
  --log-num-zeros-in-grad: true
  --log-validation-ppl-to-tensorboard: true
  --log-timers-to-tensorboard: true
  --tensorboard-dir: ${TENSORBOARD_PATH}
  --micro-batch-size: 4
  --global-batch-size: 32
  --seq-length: 1024
  --max-position-embeddings: 1024
  --train-iters: 100
  --timing-log-level: 2
  --lr-decay-iters: 320000
  --save: ${CHECKPOINT_PATH}
  --load: ${CHECKPOINT_PATH}
  --data-path: ${DATA_PATH}/my-gpt3_00_text_document
  --vocab-file: ${DATA_PATH}/bpe/vocab.json
  --merge-file: ${DATA_PATH}/bpe/merges.txt
  --split: 949,50,1
  --distributed-backend: nccl
  --lr: 0.00015
  --lr-decay-style: cosine
  --min-lr: 1.0e-5
  --weight-decay: 1e-2
  --clip-grad: 1.0
  --lr-warmup-fraction: .01
  --log-interval: 1
  --save-interval: 50
  --eval-interval: 1000
  --eval-iters: 10
  --transformer-impl: transformer_engine
  --tensor-model-parallel-size: 4
  --pipeline-model-parallel-size: 1
  --qk-layernorm: true
  --test-mode: true
  --deterministic-mode: true
  --no-gradient-accumulation-fusion: true
  --attention-softmax-in-fp32: true
  --use-checkpoint-opt_param-scheduler: true
  --use-mcore-models: true
  --ckpt-format: torch_dist
  --data-cache-path: ${DATA_CACHE_PATH}
  --bf16: true
TEST_TYPE: ckpt-resume
