feat: add rust based tokenizer

4f6f63cd · Biswa Panda · GitHub · 53163693 · 4f6f63cd · 4f6f63cd
Commit 4f6f63cd authored Feb 24, 2025 by Biswa Panda Committed by GitHub Feb 24, 2025
10 changed files
--- a/llm/rust/triton-llm/tests/snapshots/preprocessor__mulit_turn_with_system@meta_llama_llama_3_1_70b_instruct__1605565_e45e5991.snap
+++ b/llm/rust/triton-llm/tests/snapshots/preprocessor__mulit_turn_with_system@meta_llama_llama_3_1_70b_instruct__1605565_e45e5991.snap
--- a/llm/rust/triton-llm/tests/snapshots/preprocessor__mulit_turn_with_system_with_tools@meta_llama_llama_3_1_70b_instruct__1605565_e45e5991.snap
+++ b/llm/rust/triton-llm/tests/snapshots/preprocessor__mulit_turn_with_system_with_tools@meta_llama_llama_3_1_70b_instruct__1605565_e45e5991.snap
--- a/llm/rust/triton-llm/tests/snapshots/preprocessor__mulit_turn_without_system@meta_llama_llama_3_1_70b_instruct__1605565_e45e5991.snap
+++ b/llm/rust/triton-llm/tests/snapshots/preprocessor__mulit_turn_without_system@meta_llama_llama_3_1_70b_instruct__1605565_e45e5991.snap
--- a/llm/rust/triton-llm/tests/snapshots/preprocessor__multi_turn_with_continuation@meta_llama_llama_3_1_70b_instruct__1605565_e45e5991.snap
+++ b/llm/rust/triton-llm/tests/snapshots/preprocessor__multi_turn_with_continuation@meta_llama_llama_3_1_70b_instruct__1605565_e45e5991.snap
--- a/llm/rust/triton-llm/tests/snapshots/preprocessor__multi_turn_with_system_with_tools@meta_llama_llama_3_1_70b_instruct__1605565_e45e5991.snap
+++ b/llm/rust/triton-llm/tests/snapshots/preprocessor__multi_turn_with_system_with_tools@meta_llama_llama_3_1_70b_instruct__1605565_e45e5991.snap
--- a/llm/rust/triton-llm/tests/snapshots/preprocessor__single_turn@meta_llama_llama_3_1_70b_instruct__1605565_e45e5991.snap
+++ b/llm/rust/triton-llm/tests/snapshots/preprocessor__single_turn@meta_llama_llama_3_1_70b_instruct__1605565_e45e5991.snap
--- a/llm/rust/triton-llm/tests/snapshots/preprocessor__single_turn_with_tools@meta_llama_llama_3_1_70b_instruct__1605565_e45e5991.snap
+++ b/llm/rust/triton-llm/tests/snapshots/preprocessor__single_turn_with_tools@meta_llama_llama_3_1_70b_instruct__1605565_e45e5991.snap
--- a/llm/rust/triton-llm/tests/tokenizers.rs
+++ b/llm/rust/triton-llm/tests/tokenizers.rs
--- a/python-wheel/Cargo.lock
+++ b/python-wheel/Cargo.lock
--- a/python-wheel/pyproject.toml
+++ b/python-wheel/pyproject.toml