single.sh 3.02 KB
Newer Older
liangjing's avatar
v1  
liangjing committed
1
#!/bin/bash
liangjing's avatar
liangjing committed
2
3

# Runs the "7B" parameter model
liangjing's avatar
v1  
liangjing committed
4
5
export HSA_FORCE_FINE_GRAIN_PCIE=1
export OMP_NUM_THREADS=1
liangjing's avatar
liangjing committed
6
7
8
9
10
export NCCL_P2P_LEVEL=SYS

export NCCL_ALGO=Ring
export NCCL_IB_TIMEOUT=22
export CUDA_DEVICE_MAX_CONNECTIONS=1
liangjing's avatar
v1  
liangjing committed
11
12
13
14
15

lrank=$OMPI_COMM_WORLD_LOCAL_RANK
RANK=$OMPI_COMM_WORLD_RANK
WORLD_SIZE=$OMPI_COMM_WORLD_SIZE

liangjing's avatar
liangjing committed
16
17
18
19
20
CHECKPOINT_PATH=./tmp #$1 #<Specify path>
TENSORBOARD_LOGS_PATH=./tmp  #$2 #<Specify path>
DATA_PATH="./dataset/my-gpt2_text_document" #<Specify path and file prefix>_text_document
VOCAB_PATH=./gpt2-vocab.json
MERGE_PATH=./gpt2-merges.txt
liangjing's avatar
v1  
liangjing committed
21

liangjing's avatar
liangjing committed
22
23
24
25
26
27
28
29
GPT_MODEL_ARGS=(
    --num-layers 32 
    --hidden-size 4096
    --num-attention-heads 32
    --ffn-hidden-size 11008
    --seq-length 4096 
    --max-position-embeddings 4096
)
liangjing's avatar
v1  
liangjing committed
30

liangjing's avatar
liangjing committed
31
32
33
34
35
36
37
38
39
40
41
42
43
44
TRAINING_ARGS=(
    --log-throughput
    --transformer-impl local
    --use-legacy-models 
    --micro-batch-size 1 
    --global-batch-size 240 
    --train-iters 5 
    --weight-decay 0.1 
    --adam-beta1 0.9 
    --adam-beta2 0.95 
    --init-method-std 0.006 
    --clip-grad 1.0 
    --bf16
    --use-distributed-optimizer 
liangjing's avatar
update  
liangjing committed
45
    --use-flash-attn
liangjing's avatar
liangjing committed
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
    --disable-bias-linear
    --attention-dropout 0
    --hidden-dropout 0
    --no-gradient-accumulation-fusion
    --swiglu
    --lr 3.0e-5 
    --lr-decay-style cosine 
    --min-lr 3.0e-6
    --lr-warmup-iters 1
)
MODEL_PARALLEL_ARGS=(
        --sequence-parallel
	--tensor-model-parallel-size 1 
	--pipeline-model-parallel-size 4 
)
liangjing's avatar
v1  
liangjing committed
61

liangjing's avatar
liangjing committed
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
DATA_ARGS=(
    --data-path $DATA_PATH 
    --split 949,50,1
    --untie-embeddings-and-output-weights
    --use-rotary-position-embeddings 
    --normalization RMSNorm 
    --no-position-embedding
    --vocab-file $VOCAB_PATH 
    --merge-file $MERGE_PATH
    --tokenizer-type GPT2BPETokenizer 
)

EVAL_AND_LOGGING_ARGS=(
    --log-interval 1
    --save-interval 10000 
    --eval-interval 1000 
    --save $CHECKPOINT_PATH 
    --load $CHECKPOINT_PATH 
    --eval-iters 10
    --tensorboard-dir $TENSORBOARD_LOGS_PATH 
)

APP="python3  -u pretrain_gpt.py \
     ${GPT_MODEL_ARGS[@]} \
     ${TRAINING_ARGS[@]} \
     ${MODEL_PARALLEL_ARGS[@]} \
     ${DATA_ARGS[@]} \
     ${EVAL_AND_LOGGING_ARGS[@]}
     --rank ${RANK} \
     --world_size ${WORLD_SIZE} \
     --dist_url tcp://${1}:34566 \
    "
liangjing's avatar
v1  
liangjing committed
94

liangjing's avatar
update  
liangjing committed
95
#for intel cpu
liangjing's avatar
v1  
liangjing committed
96
97
case ${lrank} in
[0])
liangjing's avatar
liangjing committed
98
99
  export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
  numactl --cpunodebind=0 --membind=0 ${APP}
liangjing's avatar
v1  
liangjing committed
100
101
  ;;
[1])
liangjing's avatar
liangjing committed
102
  export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
liangjing's avatar
update  
liangjing committed
103
  numactl --cpunodebind=0 --membind=0 ${APP}
liangjing's avatar
v1  
liangjing committed
104
105
  ;;
[2])
liangjing's avatar
liangjing committed
106
  export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
liangjing's avatar
update  
liangjing committed
107
  numactl --cpunodebind=0 --membind=0 ${APP}
liangjing's avatar
v1  
liangjing committed
108
109
  ;;
[3])
liangjing's avatar
liangjing committed
110
  export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
liangjing's avatar
update  
liangjing committed
111
  numactl --cpunodebind=0 --membind=0 ${APP}
liangjing's avatar
liangjing committed
112
113
114
  ;;
[4])
  export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
liangjing's avatar
update  
liangjing committed
115
  numactl --cpunodebind=1 --membind=1 ${APP}
liangjing's avatar
liangjing committed
116
117
118
  ;;
[5])
  export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
liangjing's avatar
update  
liangjing committed
119
  numactl --cpunodebind=1 --membind=1 ${APP}
liangjing's avatar
liangjing committed
120
121
122
  ;;
[6])
  export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
liangjing's avatar
update  
liangjing committed
123
  numactl --cpunodebind=1 --membind=1 ${APP}
liangjing's avatar
liangjing committed
124
125
126
  ;;
[7])
  export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
liangjing's avatar
update  
liangjing committed
127
  numactl --cpunodebind=1 --membind=1 ${APP}
liangjing's avatar
v1  
liangjing committed
128
129
  ;;
esac
liangjing's avatar
liangjing committed
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149