train.sh

export CUDA_VISIBLE_DEVICES=1,2
nproc=2
model_name="/home/starcoder2/starcoder2-7b/"
dataset_name="/home/starcoder2/the-stack-smol/"
accelerate launch finetune.py \
        --model_id $model_name \
        --dataset_name $dataset_name \
        --subset "data/rust" \
        --dataset_text_field "content" \
        --split "train" \
        --max_seq_length 1024 \
        --max_steps 10000 \
        --micro_batch_size 1 \
        --gradient_accumulation_steps 8 \
        --learning_rate 2e-5 \
        --warmup_steps 20 \
        --num_proc "$(nproc)"