ENV=dsw
MEGATRON_PATH=../..
MODEL_SIZE=0.5B
BATCH_SIZE=1
GLOBAL_BATCH_SIZE=4
LR=1e-5
MIN_LR=1e-6
SEQ_LEN=128
PAD_LEN=128
EXTRA_VOCAB_SIZE=1
PR=fp16
TP=2
PP=1
AC=sel
DO=true
FL=true
SP=true
TE=false
SAVE_INTERVAL=100000
DATASET_PATH=/public/new-pai/data/qwen-datasets/wudao_qwenbpe_text_document
PRETRAIN_CHECKPOINT_PATH=/public/new-pai/model/Qwen1.5-0.5B
TRAIN_TOKENS=100000
WARMUP_TOKENS=10000
OUTPUT_BASEPATH=./output

rm -rf $OUTPUT_BASEPATH
sh run_pretrain_megatron_qwen-dcu.sh $ENV $MEGATRON_PATH $MODEL_SIZE $BATCH_SIZE $GLOBAL_BATCH_SIZE $LR $MIN_LR $SEQ_LEN $PAD_LEN $EXTRA_VOCAB_SIZE $PR $TP $PP $AC $DO $FL $SP $TE $SAVE_INTERVAL $DATASET_PATH $PRETRAIN_CHECKPOINT_PATH $TRAIN_TOKENS $WARMUP_TOKENS $OUTPUT_BASEPATH