"tools/vscode:/vscode.git/clone" did not exist on "505e0ab31dacfdfeda73dbc437d24b3c958a0397"
train_bert_test.sh 1008 Bytes
Newer Older
Guolin Ke's avatar
Guolin Ke committed
1
2
3
4
[ -z "${MASTER_PORT}" ] && MASTER_PORT=10086
[ -z "${n_gpu}" ] && n_gpu=$(nvidia-smi -L | wc -l)
export NCCL_ASYNC_ERROR_HANDLING=1
export OMP_NUM_THREADS=1
Guolin Ke's avatar
Guolin Ke committed
5
6
7
run_name=bert_example
save_dir="./save/${run_name}"
mkdir -p ${save_dir}
robotcator's avatar
robotcator committed
8
9

torchrun --standalone --nnodes=1 --nproc_per_node=$n_gpu $(which unicore-train) ./example_data  --user-dir . --valid-subset valid \
Guolin Ke's avatar
Guolin Ke committed
10
11
12
13
14
       --num-workers 0 --ddp-backend=c10d \
       --task bert --loss masked_lm --arch bert_base  \
       --optimizer adam --adam-betas '(0.9, 0.98)' --adam-eps 1e-6 --clip-norm 1.0 \
       --lr-scheduler polynomial_decay --lr 1e-4 --warmup-updates 100 --total-num-update 10000 --batch-size 4 \
       --update-freq 1 --seed 1 \
Guolin Ke's avatar
Guolin Ke committed
15
       --fp16 --fp16-init-scale 4 --fp16-scale-window 256 --tensorboard-logdir $save_dir/tsb \
Guolin Ke's avatar
Guolin Ke committed
16
       --max-update 10000 --log-interval 100 --log-format simple \
Guolin Ke's avatar
Guolin Ke committed
17
18
       --save-interval-updates 1000 --validate-interval-updates 1000 --keep-interval-updates 30 --no-epoch-checkpoints  \
       --save-dir $save_dir
Guolin Ke's avatar
Guolin Ke committed
19
20