feat: add more useful APIs for tokens (#313)

Add `AsMut`, `DerefMut` and `IntoIterator` trait impl for the `Tokens` structure.
Signed-off-by: nora-coder-dot <nora6677@gmail.com>
Co-authored-by: nora-coder-dot <nora6677@gmail.com>

lib/llm/src/tokens.rs

@@ -35,6 +35,12 @@ impl AsRef<[Token]> for Tokens {
     }
 }
 
+impl AsMut<[Token]> for Tokens {
+    fn as_mut(&mut self) -> &mut [Token] {
+        &mut self.0
+    }
+}
+
 impl std::ops::Deref for Tokens {
     type Target = [Token];
 
@@ -43,12 +49,28 @@ impl std::ops::Deref for Tokens {
     }
 }
 
+impl std::ops::DerefMut for Tokens {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+
 impl std::borrow::Borrow<[Token]> for Tokens {
     fn borrow(&self) -> &[Token] {
         &self.0
     }
 }
 
+impl IntoIterator for Tokens {
+    type Item = Token;
+
+    type IntoIter = std::vec::IntoIter<Token>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.0.into_iter()
+    }
+}
+
 impl From<Vec<Token>> for Tokens {
     fn from(tokens: Vec<Token>) -> Self {
         Tokens(tokens)