Merge branch 'master' into 'master'

ChatGLM3-6B 微调训练

See merge request !2

finetune_chatmodel_demo/requirements.txt

-transformers==4.30.2
-accelerate
-sentencepiece
-astunparse
-deepspeed
\ No newline at end of file

finetune_chatmodel_demo/scripts/finetune_ds.sh

-#! /usr/bin/env bash
-
-set -ex
-
-LR=1e-4
-NUM_GPUS=8
-MAX_SOURCE_LEN=1024
-MAX_TARGET_LEN=128
-DEV_BATCH_SIZE=4
-GRAD_ACCUMULARION_STEPS=1
-MAX_STEP=20
-SAVE_INTERVAL=500
-
-RUN_NAME=advertise_gen_ft
-BASE_MODEL_PATH=/chatglm3/chatglm3-6b
-DATASET_PATH=../formatted_data/advertise_gen.jsonl
-
-DATESTR=`date +%Y%m%d-%H%M%S`
-OUTPUT_DIR=output/${RUN_NAME}-${DATESTR}-${LR}
-MASTER_PORT=$(shuf -n 1 -i 10000-65535)
-
-mkdir -p $OUTPUT_DIR
-export HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
-torchrun --standalone --nnodes=1 --nproc_per_node=$NUM_GPUS ../finetune.py \
-    --train_format input-output \
-    --train_file $DATASET_PATH \
-    --preprocessing_num_workers 1 \
-    --model_name_or_path $BASE_MODEL_PATH \
-    --output_dir $OUTPUT_DIR \
-    --max_source_length $MAX_SOURCE_LEN \
-    --max_target_length $MAX_TARGET_LEN \
-    --per_device_train_batch_size $DEV_BATCH_SIZE \
-    --gradient_accumulation_steps $GRAD_ACCUMULARION_STEPS \
-    --max_steps $MAX_STEP \
-    --logging_steps 1 \
-    --save_steps $SAVE_INTERVAL \
-    --learning_rate $LR \
-    --fp16 \
-    --deepspeed ../configs/deepspeed.json 2>&1 | tee ${OUTPUT_DIR}/train.log

finetune_chatmodel_demo/scripts/format_advertise_gen.py

-#! /usr/bin/env python
-
-import json
-from collections import Counter
-from argparse import ArgumentParser
-import os
-
-parser = ArgumentParser()
-parser.add_argument("--path", type=str, required=True)
-
-args = parser.parse_args()
-
-with open(args.path) as f:
-    data = [json.loads(line) for line in f]
-
-train_examples = [{
-    "prompt": x['content'],
-    "response": x['summary'],
-} for x in data]
-
-os.makedirs("formatted_data", exist_ok=True)
-
-with open("formatted_data/advertise_gen.jsonl", "w") as f:
-    for e in train_examples:
-        f.write(json.dumps(e, ensure_ascii=False) + "\n")

finetune_demo/lora.sh

+export HIP_VISIBLE_DEVICES=7
+python finetune_hf.py  data/AdvertiseGen_fix  /path/to/chatglm3-6b  configs/lora.yaml