v1.0.1

convert_data_to_ds.sh

+python3 tools/preprocess_data.py \
+       --tokenizer-name-or-path HuggingFaceTB/cosmo2-tokenizer \
+       --output-folder datasets/fineweb-edu-dedup-ds \
+       --n-tasks 16 \
+       hf \
+       --dataset datasets/fineweb-edu-dedup \

convert_hf_to_nanotron.sh

+torchrun --nproc-per-node 1 examples/llama/convert_hf_to_nanotron.py --checkpoint_path Meta-Llama-3.1-8B --save_path checkpoints/Nanotron-Llama-3.1-8B
+# torchrun --nproc-per-node 1 examples/llama/convert_hf_to_nanotron.py --checkpoint_path Llama-3.2-3B --save_path checkpoints/Nanotron-Llama-3.2-3B

convert_nanotron_to_hf.sh

+torchrun --nproc-per-node 1 examples/llama/convert_nanotron_to_hf.py --checkpoint_path checkpoints/10 --save_path hf/hf-Llama-3.1-8B --tokenizer_name Meta-Llama-3.1-8B