Unverified Commit dc01313d authored by Chang Su's avatar Chang Su Committed by GitHub
Browse files

[router] Add rustfmt and set group imports by default (#11732)

parent 7a7f99be
use super::traits::{TokenIdType, Tokenizer as TokenizerTrait};
use anyhow::Result;
use std::sync::Arc; use std::sync::Arc;
use anyhow::Result;
use super::traits::{TokenIdType, Tokenizer as TokenizerTrait};
/// Maintains state for an ongoing sequence of tokens and their decoded text /// Maintains state for an ongoing sequence of tokens and their decoded text
/// This provides a cleaner abstraction for managing token sequences /// This provides a cleaner abstraction for managing token sequences
pub struct Sequence { pub struct Sequence {
......
use super::sequence::Sequence; use std::{collections::HashSet, sync::Arc};
use super::traits::{self, TokenIdType};
use anyhow::Result; use anyhow::Result;
use std::collections::HashSet;
use std::sync::Arc; use super::{
sequence::Sequence,
traits::{self, TokenIdType},
};
/// Output from the sequence decoder /// Output from the sequence decoder
#[derive(Debug, Clone, PartialEq)] #[derive(Debug, Clone, PartialEq)]
......
// src/tokenizer/stream.rs // src/tokenizer/stream.rs
use super::traits::{self, TokenIdType};
use anyhow::Result;
use std::sync::Arc; use std::sync::Arc;
use anyhow::Result;
use super::traits::{self, TokenIdType};
const INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET: usize = 5; const INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET: usize = 5;
/// DecodeStream will keep the state necessary to produce individual chunks of /// DecodeStream will keep the state necessary to produce individual chunks of
......
#[cfg(test)] #[cfg(test)]
use super::*;
#[cfg(test)]
use std::sync::Arc; use std::sync::Arc;
#[cfg(test)]
use super::*;
#[test] #[test]
fn test_mock_tokenizer_encode() { fn test_mock_tokenizer_encode() {
let tokenizer = mock::MockTokenizer::new(); let tokenizer = mock::MockTokenizer::new();
......
use anyhow::{Error, Result};
use tiktoken_rs::{cl100k_base, p50k_base, p50k_edit, r50k_base, CoreBPE};
use super::traits::{ use super::traits::{
Decoder, Encoder, Encoding, SpecialTokens, TokenIdType, Tokenizer as TokenizerTrait, Decoder, Encoder, Encoding, SpecialTokens, TokenIdType, Tokenizer as TokenizerTrait,
}; };
use anyhow::{Error, Result};
use tiktoken_rs::{cl100k_base, p50k_base, p50k_edit, r50k_base, CoreBPE};
/// Tiktoken tokenizer wrapper for OpenAI GPT models /// Tiktoken tokenizer wrapper for OpenAI GPT models
pub struct TiktokenTokenizer { pub struct TiktokenTokenizer {
......
use std::{
collections::hash_map::DefaultHasher,
hash::{Hash, Hasher},
};
use anyhow::Result; use anyhow::Result;
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
/// Type alias for token IDs /// Type alias for token IDs
pub type TokenIdType = u32; pub type TokenIdType = u32;
......
// Factory and pool for creating model-specific tool parsers with pooling support. // Factory and pool for creating model-specific tool parsers with pooling support.
use std::collections::HashMap; use std::{
use std::sync::{Arc, RwLock}; collections::HashMap,
sync::{Arc, RwLock},
};
use tokio::sync::Mutex; use tokio::sync::Mutex;
use crate::tool_parser::parsers::{ use crate::tool_parser::{
DeepSeekParser, Glm4MoeParser, GptOssHarmonyParser, GptOssParser, JsonParser, KimiK2Parser, parsers::{
LlamaParser, MistralParser, PassthroughParser, PythonicParser, QwenParser, Step3Parser, DeepSeekParser, Glm4MoeParser, GptOssHarmonyParser, GptOssParser, JsonParser, KimiK2Parser,
LlamaParser, MistralParser, PassthroughParser, PythonicParser, QwenParser, Step3Parser,
},
traits::ToolParser,
}; };
use crate::tool_parser::traits::ToolParser;
/// Type alias for pooled parser instances. /// Type alias for pooled parser instances.
pub type PooledParser = Arc<Mutex<Box<dyn ToolParser>>>; pub type PooledParser = Arc<Mutex<Box<dyn ToolParser>>>;
......
...@@ -18,11 +18,10 @@ mod tests; ...@@ -18,11 +18,10 @@ mod tests;
// Re-export commonly used types // Re-export commonly used types
pub use errors::{ParserError, ParserResult}; pub use errors::{ParserError, ParserResult};
pub use factory::{ParserFactory, ParserRegistry, PooledParser}; pub use factory::{ParserFactory, ParserRegistry, PooledParser};
pub use traits::{PartialJsonParser, ToolParser};
pub use types::{FunctionCall, PartialToolCall, StreamingParseResult, ToolCall};
// Re-export parsers for convenience // Re-export parsers for convenience
pub use parsers::{ pub use parsers::{
DeepSeekParser, Glm4MoeParser, GptOssParser, JsonParser, KimiK2Parser, LlamaParser, DeepSeekParser, Glm4MoeParser, GptOssParser, JsonParser, KimiK2Parser, LlamaParser,
MistralParser, PythonicParser, QwenParser, Step3Parser, MistralParser, PythonicParser, QwenParser, Step3Parser,
}; };
pub use traits::{PartialJsonParser, ToolParser};
pub use types::{FunctionCall, PartialToolCall, StreamingParseResult, ToolCall};
...@@ -2,13 +2,14 @@ use async_trait::async_trait; ...@@ -2,13 +2,14 @@ use async_trait::async_trait;
use regex::Regex; use regex::Regex;
use serde_json::Value; use serde_json::Value;
use crate::protocols::common::Tool; use crate::{
protocols::common::Tool,
use crate::tool_parser::{ tool_parser::{
errors::{ParserError, ParserResult}, errors::{ParserError, ParserResult},
parsers::helpers, parsers::helpers,
traits::ToolParser, traits::ToolParser,
types::{FunctionCall, StreamingParseResult, ToolCall, ToolCallItem}, types::{FunctionCall, StreamingParseResult, ToolCall, ToolCallItem},
},
}; };
/// DeepSeek V3 format parser for tool calls /// DeepSeek V3 format parser for tool calls
......
...@@ -2,13 +2,14 @@ use async_trait::async_trait; ...@@ -2,13 +2,14 @@ use async_trait::async_trait;
use regex::Regex; use regex::Regex;
use serde_json::Value; use serde_json::Value;
use crate::protocols::common::Tool; use crate::{
protocols::common::Tool,
use crate::tool_parser::{ tool_parser::{
errors::{ParserError, ParserResult}, errors::{ParserError, ParserResult},
parsers::helpers, parsers::helpers,
traits::ToolParser, traits::ToolParser,
types::{FunctionCall, StreamingParseResult, ToolCall, ToolCallItem}, types::{FunctionCall, StreamingParseResult, ToolCall, ToolCallItem},
},
}; };
/// GLM-4 MoE format parser for tool calls /// GLM-4 MoE format parser for tool calls
......
use async_trait::async_trait; use async_trait::async_trait;
use crate::protocols::common::Tool; use crate::{
protocols::common::Tool,
use crate::tool_parser::{ tool_parser::{
errors::ParserResult, errors::ParserResult,
traits::{TokenToolParser, ToolParser}, traits::{TokenToolParser, ToolParser},
types::{StreamingParseResult, ToolCall}, types::{StreamingParseResult, ToolCall},
},
}; };
/// Placeholder for the Harmony-backed GPT-OSS parser. /// Placeholder for the Harmony-backed GPT-OSS parser.
......
...@@ -2,14 +2,15 @@ use async_trait::async_trait; ...@@ -2,14 +2,15 @@ use async_trait::async_trait;
use regex::Regex; use regex::Regex;
use serde_json::Value; use serde_json::Value;
use crate::protocols::common::Tool; use crate::{
protocols::common::Tool,
use crate::tool_parser::{ tool_parser::{
errors::{ParserError, ParserResult}, errors::{ParserError, ParserResult},
parsers::helpers, parsers::helpers,
partial_json::PartialJson, partial_json::PartialJson,
traits::ToolParser, traits::ToolParser,
types::{FunctionCall, StreamingParseResult, ToolCall, ToolCallItem}, types::{FunctionCall, StreamingParseResult, ToolCall, ToolCallItem},
},
}; };
/// GPT-OSS format parser for tool calls /// GPT-OSS format parser for tool calls
......
use crate::protocols::common::Tool;
use serde_json::Value;
use std::collections::HashMap; use std::collections::HashMap;
use crate::tool_parser::errors::{ParserError, ParserResult}; use serde_json::Value;
use crate::tool_parser::types::{StreamingParseResult, ToolCallItem};
use crate::{
protocols::common::Tool,
tool_parser::{
errors::{ParserError, ParserResult},
types::{StreamingParseResult, ToolCallItem},
},
};
/// Get a mapping of tool names to their indices /// Get a mapping of tool names to their indices
pub fn get_tool_indices(tools: &[Tool]) -> HashMap<String, usize> { pub fn get_tool_indices(tools: &[Tool]) -> HashMap<String, usize> {
......
use async_trait::async_trait; use async_trait::async_trait;
use serde_json::Value; use serde_json::Value;
use crate::protocols::common::Tool; use crate::{
protocols::common::Tool,
use crate::tool_parser::{ tool_parser::{
errors::{ParserError, ParserResult}, errors::{ParserError, ParserResult},
parsers::helpers, parsers::helpers,
partial_json::PartialJson, partial_json::PartialJson,
traits::ToolParser, traits::ToolParser,
types::{FunctionCall, StreamingParseResult, ToolCall, ToolCallItem}, types::{FunctionCall, StreamingParseResult, ToolCall, ToolCallItem},
},
}; };
/// JSON format parser for tool calls /// JSON format parser for tool calls
......
...@@ -2,13 +2,14 @@ use async_trait::async_trait; ...@@ -2,13 +2,14 @@ use async_trait::async_trait;
use regex::Regex; use regex::Regex;
use serde_json::Value; use serde_json::Value;
use crate::protocols::common::Tool; use crate::{
protocols::common::Tool,
use crate::tool_parser::{ tool_parser::{
errors::ParserResult, errors::ParserResult,
parsers::helpers, parsers::helpers,
traits::ToolParser, traits::ToolParser,
types::{FunctionCall, StreamingParseResult, ToolCall, ToolCallItem}, types::{FunctionCall, StreamingParseResult, ToolCall, ToolCallItem},
},
}; };
/// Kimi K2 format parser for tool calls /// Kimi K2 format parser for tool calls
......
use async_trait::async_trait; use async_trait::async_trait;
use serde_json::Value; use serde_json::Value;
use crate::protocols::common::Tool; use crate::{
protocols::common::Tool,
use crate::tool_parser::{ tool_parser::{
errors::{ParserError, ParserResult}, errors::{ParserError, ParserResult},
parsers::helpers, parsers::helpers,
partial_json::PartialJson, partial_json::PartialJson,
traits::ToolParser, traits::ToolParser,
types::{FunctionCall, StreamingParseResult, ToolCall}, types::{FunctionCall, StreamingParseResult, ToolCall},
},
}; };
/// Llama 3.2 format parser for tool calls /// Llama 3.2 format parser for tool calls
......
use async_trait::async_trait; use async_trait::async_trait;
use serde_json::Value; use serde_json::Value;
use crate::protocols::common::Tool; use crate::{
protocols::common::Tool,
use crate::tool_parser::{ tool_parser::{
errors::{ParserError, ParserResult}, errors::{ParserError, ParserResult},
parsers::helpers, parsers::helpers,
partial_json::PartialJson, partial_json::PartialJson,
traits::ToolParser, traits::ToolParser,
types::{FunctionCall, StreamingParseResult, ToolCall}, types::{FunctionCall, StreamingParseResult, ToolCall},
},
}; };
/// Mistral format parser for tool calls /// Mistral format parser for tool calls
......
...@@ -4,12 +4,17 @@ ...@@ -4,12 +4,17 @@
//! tool call parsing should be performed. It simply returns the input text //! tool call parsing should be performed. It simply returns the input text
//! with no tool calls detected. //! with no tool calls detected.
use crate::protocols::common::Tool;
use crate::tool_parser::errors::ParserResult;
use crate::tool_parser::traits::ToolParser;
use crate::tool_parser::types::{StreamingParseResult, ToolCall, ToolCallItem};
use async_trait::async_trait; use async_trait::async_trait;
use crate::{
protocols::common::Tool,
tool_parser::{
errors::ParserResult,
traits::ToolParser,
types::{StreamingParseResult, ToolCall, ToolCallItem},
},
};
/// Passthrough parser that returns text unchanged with no tool calls /// Passthrough parser that returns text unchanged with no tool calls
#[derive(Default)] #[derive(Default)]
pub struct PassthroughParser; pub struct PassthroughParser;
......
use std::sync::OnceLock;
/// Pythonic format parser for tool calls /// Pythonic format parser for tool calls
/// ///
/// Handles Python function call syntax within square brackets: /// Handles Python function call syntax within square brackets:
...@@ -10,18 +12,20 @@ ...@@ -10,18 +12,20 @@
use async_trait::async_trait; use async_trait::async_trait;
use num_traits::ToPrimitive; use num_traits::ToPrimitive;
use regex::Regex; use regex::Regex;
use rustpython_parser::ast::{Constant, Expr, Mod, UnaryOp}; use rustpython_parser::{
use rustpython_parser::{parse, Mode}; ast::{Constant, Expr, Mod, UnaryOp},
parse, Mode,
};
use serde_json::{Map, Number, Value}; use serde_json::{Map, Number, Value};
use std::sync::OnceLock;
use crate::protocols::common::Tool;
use crate::tool_parser::{ use crate::{
errors::{ParserError, ParserResult}, protocols::common::Tool,
parsers::helpers, tool_parser::{
traits::ToolParser, errors::{ParserError, ParserResult},
types::{FunctionCall, StreamingParseResult, ToolCall, ToolCallItem}, parsers::helpers,
traits::ToolParser,
types::{FunctionCall, StreamingParseResult, ToolCall, ToolCallItem},
},
}; };
static PYTHONIC_BLOCK_REGEX: OnceLock<Regex> = OnceLock::new(); static PYTHONIC_BLOCK_REGEX: OnceLock<Regex> = OnceLock::new();
......
...@@ -2,14 +2,15 @@ use async_trait::async_trait; ...@@ -2,14 +2,15 @@ use async_trait::async_trait;
use regex::Regex; use regex::Regex;
use serde_json::Value; use serde_json::Value;
use crate::protocols::common::Tool; use crate::{
protocols::common::Tool,
use crate::tool_parser::{ tool_parser::{
errors::{ParserError, ParserResult}, errors::{ParserError, ParserResult},
parsers::helpers, parsers::helpers,
partial_json::PartialJson, partial_json::PartialJson,
traits::ToolParser, traits::ToolParser,
types::{FunctionCall, StreamingParseResult, ToolCall}, types::{FunctionCall, StreamingParseResult, ToolCall},
},
}; };
/// Qwen format parser for tool calls /// Qwen format parser for tool calls
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment