train.sh

#!/bin/bash
# Copyright (c) 2024 Bytedance Ltd. and/or its affiliate

# 1B: 128 A100s for ≈ 0.5days
# 7B: 128 A100s for ≈ 2days
# For Books, training with a sequence length of 55 and 56*512=28672 negatives in total, serving in a sequence length of 50.
cd code && python3 main.py \
--config_file overall/LLM_deepspeed.yaml HLLM/HLLM.yaml \
--loss nce \
--epochs 5 \
--dataset Pixel200K \
--train_batch_size 8 \
--MAX_TEXT_LENGTH 256 \
--MAX_ITEM_LIST_LENGTH 10 \
--checkpoint_dir saved_path \
--optim_args.learning_rate 1e-4 \
--item_pretrain_dir TinyLlama-1.1B-Chat-v1.0 \
--user_pretrain_dir TinyLlama-1.1B-Chat-v1.0 \
--text_path "../information" \
--text_keys '[\"title\",\"tag\",\"description\"]'