Cargo.toml

# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

[package]
name = "dynamo-tokenizers"
version.workspace = true
edition.workspace = true
description = "Standalone tokenizer implementations for Dynamo"
authors.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
keywords.workspace = true

[dependencies]
anyhow = { workspace = true }
fastokens = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
strum = { workspace = true }
tracing = { workspace = true }

base64 = { version = "0.22" }
rayon = "1"
rustc-hash = "1.1"
tiktoken-rs = { version = "0.9", default-features = false }
tokenizers = { version = "0.21.4", default-features = false, features = [
  "onig",
  "esaxx_fast",
  "rustls-tls",
] }

[dev-dependencies]
tempfile = { workspace = true }