Unverified Commit a04efc49 authored by Simo Lin's avatar Simo Lin Committed by GitHub
Browse files

[router] when given both local tokenizer and chat template, log all (#11601)

parent 642fa966
...@@ -49,9 +49,9 @@ pub fn create_tokenizer_with_chat_template( ...@@ -49,9 +49,9 @@ pub fn create_tokenizer_with_chat_template(
if path.is_dir() { if path.is_dir() {
let tokenizer_json = path.join("tokenizer.json"); let tokenizer_json = path.join("tokenizer.json");
if tokenizer_json.exists() { if tokenizer_json.exists() {
let chat_template_path = chat_template_path // Resolve chat template: provided path takes precedence over auto-discovery
.map(|s| s.to_string()) let final_chat_template =
.or_else(|| discover_chat_template_in_dir(path)); resolve_and_log_chat_template(chat_template_path, path, file_path);
let tokenizer_path_str = tokenizer_json.to_str().ok_or_else(|| { let tokenizer_path_str = tokenizer_json.to_str().ok_or_else(|| {
Error::msg(format!( Error::msg(format!(
"Tokenizer path is not valid UTF-8: {:?}", "Tokenizer path is not valid UTF-8: {:?}",
...@@ -60,7 +60,7 @@ pub fn create_tokenizer_with_chat_template( ...@@ -60,7 +60,7 @@ pub fn create_tokenizer_with_chat_template(
})?; })?;
return create_tokenizer_with_chat_template( return create_tokenizer_with_chat_template(
tokenizer_path_str, tokenizer_path_str,
chat_template_path.as_deref(), final_chat_template.as_deref(),
); );
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment