Commit d99506f3 authored by chenzk's avatar chenzk
Browse files

v1.0.1

parent 61e92904
Pipeline #2033 canceled with stages
python3 tools/preprocess_data.py \
--tokenizer-name-or-path HuggingFaceTB/cosmo2-tokenizer \
--output-folder datasets/fineweb-edu-dedup-ds \
--n-tasks 16 \
hf \
--dataset datasets/fineweb-edu-dedup \
torchrun --nproc-per-node 1 examples/llama/convert_hf_to_nanotron.py --checkpoint_path Meta-Llama-3.1-8B --save_path checkpoints/Nanotron-Llama-3.1-8B
# torchrun --nproc-per-node 1 examples/llama/convert_hf_to_nanotron.py --checkpoint_path Llama-3.2-3B --save_path checkpoints/Nanotron-Llama-3.2-3B
torchrun --nproc-per-node 1 examples/llama/convert_nanotron_to_hf.py --checkpoint_path checkpoints/10 --save_path hf/hf-Llama-3.1-8B --tokenizer_name Meta-Llama-3.1-8B
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment