Merge branch 'k100ai_dtk2404_optim'

ptuning/deepspeed.json

@@ -21,14 +21,6 @@
   },
   "zero_optimization": {
     "stage": 3,
-    "offload_optimizer": {
-      "device": "cpu",
-      "pin_memory": true
-    },
-    "offload_param": {
-      "device": "cpu",
-      "pin_memory": true
-    },
     "stage3_gather_16bit_weights_on_model_save": true,
     "allgather_partitions": true,
     "allgather_bucket_size": 5e8,

ptuning/ft_train.sh

 LR=5e-5
-
+export HSA_FORCE_FINE_GRAIN_PCIE=1
+export NCCL_P2P_LEVEL=5
+export NCCL_MIN_NCHANNELS=20
 MASTER_PORT=$(shuf -n 1 -i 10000-65535)
-
 HIP_VISIBLE_DEVICES=0,1,2,3 deepspeed --num_gpus=4 --master_port $MASTER_PORT main.py \
     --deepspeed deepspeed.json \
     --do_train \
@@ -9,19 +10,17 @@ HIP_VISIBLE_DEVICES=0,1,2,3 deepspeed --num_gpus=4 --master_port $MASTER_PORT ma
     --test_file AdvertiseGen/dev.json \
     --prompt_column content \
     --response_column summary \
-    --overwrite_cache \
-    --model_name_or_path THUDM/chatglm-6b \
-    --output_dir ./output_ft/adgen-chatglm-6b-ft-4c-$LR \
+    --model_name_or_path ../model/chatglm-6b \
+    --output_dir ./output_ft/nooptim-adgen-chatglm-6b-ft-4c-$LR \
     --overwrite_output_dir \
-    --max_source_length 64 \
-    --max_target_length 64 \
-    --per_device_train_batch_size 32 \
+    --max_source_length 512 \
+    --max_target_length 512 \
+    --per_device_train_batch_size 1 \
     --per_device_eval_batch_size 1 \
     --gradient_accumulation_steps 1 \
     --predict_with_generate \
-    --max_steps 5000 \
-    --logging_steps 10 \
+    --max_steps 30 \
+    --logging_steps 30 \
     --save_steps 1000 \
     --learning_rate $LR \
-    --fp16
-
+    --fp16 \
\ No newline at end of file

ptuning/ptuning_train.sh

-PRE_SEQ_LEN=128
+PRE_SEQ_LEN=1024
 LR=5e-3
 
-MASTER_PORT=$(shuf -n 1 -i 10000-65535)
+export HSA_FORCE_FINE_GRAIN_PCIE=1
+export NCCL_P2P_LEVEL=5
+export NCCL_MIN_NCHANNELS=20
 
-HIP_VISIBLE_DEVICES=0,1,2,3 deepspeed --master_port $MASTER_PORT main.py \
-    --deepspeed deepspeed.json \
+MASTER_PORT=$(shuf -n 1 -i 10000-65535)
+HIP_VISIBLE_DEVICES=0,1,2,3 deepspeed --num_gpus=4 --master_port $MASTER_PORT main.py \
     --do_train \
     --train_file AdvertiseGen/train.json \
     --test_file AdvertiseGen/dev.json \
     --prompt_column content \
     --response_column summary \
-    --model_name_or_path THUDM/chatglm-6b \
+    --model_name_or_path ../model/chatglm-6b \
     --output_dir ./output_pt/adgen-chatglm-6b-pt-4c-$LR \
     --overwrite_output_dir \
-    --max_source_length 64 \
-    --max_target_length 64 \
-    --per_device_train_batch_size 16 \
+    --max_source_length 512 \
+    --max_target_length 512 \
+    --per_device_train_batch_size 1 \
     --per_device_eval_batch_size 1 \
     --gradient_accumulation_steps 1 \
     --predict_with_generate \
-    --max_steps 3000 \
-    --logging_steps 10 \
+    --max_steps 30 \
+    --logging_steps 1 \
     --save_steps 1000 \
     --learning_rate $LR \
     --pre_seq_len $PRE_SEQ_LEN \
-    --fp16
- 
+    --fp16 \
\ No newline at end of file