pyproject.toml 16.5 KB
Newer Older
1
# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
3
# SPDX-License-Identifier: Apache-2.0

Neelay Shah's avatar
Neelay Shah committed
4
[project]
5
name = "ai-dynamo"
6
version = "1.1.0"
Neelay Shah's avatar
Neelay Shah committed
7
8
9
description = "Distributed Inference Framework"
readme = "README.md"
authors = [
Neelay Shah's avatar
Neelay Shah committed
10
    { name = "NVIDIA Inc.", email = "sw-dl-dynamo@nvidia.com" },
Neelay Shah's avatar
Neelay Shah committed
11
]
12
13
license = { text = "Apache-2.0" }
license-files = ["LICENSE"]
Neelay Shah's avatar
Neelay Shah committed
14
15
requires-python = ">=3.10"
dependencies = [
16
    "ai-dynamo-runtime==1.1.0",
17
    "transformers>=4.56.0",
18
    "kubernetes>=32.0.1,<33.0.0",
19
    "prometheus_client>=0.23.1,<1.0",
20
21
    "msgspec>=0.19.0",
    "pyzmq>=26.0.0",
22
    "msgpack==1.1.2",
Neelay Shah's avatar
Neelay Shah committed
23
]
24

25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
classifiers = [
    "Development Status :: 4 - Beta",
    "Intended Audience :: Developers",
    "Intended Audience :: Science/Research",
    "Intended Audience :: Information Technology",
    "License :: OSI Approved :: Apache Software License",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Topic :: Scientific/Engineering",
    "Topic :: Scientific/Engineering :: Artificial Intelligence",
    "Operating System :: POSIX :: Linux",
]
keywords = ["llm", "genai", "inference", "nvidia", "distributed", "dynamo"]
Neelay Shah's avatar
Neelay Shah committed
40

41
42
43
[project.urls]
Repository = "https://github.com/ai-dynamo/dynamo.git"

44
[project.optional-dependencies]
45
46
trtllm =[
    "uvloop",
47
    "tensorrt-llm==1.3.0rc11",
48
49
]

50
51
vllm = [
    "uvloop",
52
    "nixl[cu12]<=0.10.1",
53
    "vllm[flashinfer,runai,otel]==0.19.1",
Alec's avatar
Alec committed
54
55
56
57
    # vllm-omni is installed separately in container builds (see
    # container/deps/vllm/install_vllm.sh). Do not add it to ai-dynamo[vllm]:
    # pip/uv dependency resolution for omni can override the vLLM torch stack.
    # "vllm-omni==...",
58
    "blake3>=1.0.0,<2.0.0",
59
60
    "soundfile>=0.13.1",
    "librosa>=0.10.0",
61
    "ray==2.55.0",
62
63
]

64
65
sglang = [
    "uvloop",
66
    "sglang[diffusion]==0.5.10.post1",
67
    "nixl[cu12]>=1.0.0,<1.1.0",
68
    "cupy-cuda12x>=13.0.0",
69
70
]

71
72
73
74
mocker = [
    "aiconfigurator>=0.7.0",
]

75
76
77
78
79
[project.entry-points.pytest11]
vllm_tests = "dynamo.vllm.tests.conftest"
trtllm_tests = "dynamo.trtllm.tests.conftest"
sglang_tests = "dynamo.sglang.tests.conftest"

80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
[dependency-groups]
docs = [
    # Core Sphinx
    "sphinx>=8.1",
    "nvidia-sphinx-theme>=0.0.8",
    # Sphinx extensions
    "ablog>=0.11",
    "sphinx-copybutton>=0.5",
    "sphinx-design>=0.6",
    "sphinx-prompt>=1.9",
    "sphinx-sitemap>=2.6",
    "sphinx-tabs>=3.4",
    "sphinx-book-theme>=1.1",
    "sphinxcontrib-mermaid>=1.0",
    "sphinxcontrib-bibtex>=2.6",
95
    "sphinx-reredirects>=1.0.0",
96
97
98
99
100
101
    # Markdown and notebook support
    "myst-parser>=4.0",
    "myst-nb>=1.2",
    "nbsphinx>=0.9",
]

102
[build-system]
103
104
105
requires = ["hatchling"]
build-backend = "hatchling.build"

106
107
108
[tool.hatch.build.hooks.custom]
path = "hatch_build.py"

109
[tool.hatch.build.targets.wheel]
110
packages = [
111
    "components/src/dynamo",
112
]
113

114
115
116
[tool.hatch.metadata]
allow-direct-references = true

117
118
119
120
121
[tool.codespell]
# note: pre-commit passes explicit lists of files here, which this skip file list doesn't override -
# this is only to allow you to run codespell interactively
# this also overrides the grpc_generated folder, since it is generated

122
123
# Ignore data files and auto-generated files
skip = "./.git,./.github,./lib/llm/tests/data,*.lock,*.sum"
124

125
126
# ignore allowed words used in code
ignore-words-list = "afterall,ser,ende"
127
128
# use the 'clear' dictionary for unambiguous spelling mistakes
builtin = "clear"
129
130
# use custom dictionary in addition to the built-in one
dictionary = "./codespell.txt"
131
132
133
134
135
136
137
138
139
140
141
142
143
144
# disable warnings about binary files and wrong encoding
quiet-level = 3

[tool.isort]
profile = "black"
use_parentheses = true
multi_line_output = 3
include_trailing_comma = true
force_grid_wrap = 0
ensure_newline_before_comments = true
line_length = 88
balanced_wrapping = true
indent = "    "
skip = ["build"]
145
known_first_party = ["dynamo", "deploy"]
146
147
148
149
150
151
# isort may confuse what is 1st or 3rd library. e.g.
# when dynamo/vllm/omni/xx.py import vllm, local isort may treat this `vllm` as first
# party heuristically. This causes local sort differs from GitHub sort and pre-commit
# failure. To mitigate 1) one can install 3rd party lib so that isort is aware of it,
# 2) hardcode 3rd party lib here, 3) add "# isort: skip_file" to problematic files
# as the last resort.
152
known_third_party = ["vllm", "tensorrt_llm", "sglang", "aiconfigurator"]
153
154
155

[tool.pytest.ini_options]
minversion = "8.0"
156
tmp_path_retention_policy = "failed"
Neelay Shah's avatar
Neelay Shah committed
157
158

# NOTE
jh-nv's avatar
jh-nv committed
159
160
# Keep these ignores in pytest collection to avoid duplicate-module collection
# errors (for example, backend trees that include multiple model.py files).
161
162
163
164
165
166
addopts = [
    "-ra",
    "--showlocals",
    "--strict-markers",
    "--strict-config",
    "--ignore-glob=*model.py",
Ryan Olson's avatar
Ryan Olson committed
167
    "--ignore-glob=*vllm_integration*",
168
    "--ignore-glob=*trtllm_integration*",
Richard Huo's avatar
Richard Huo committed
169
    "--ignore-glob=*kvbm/python/kvbm*",
170
    "--ignore-glob=*_inc.py",
171
    "--ignore-glob=*/llm/tensorrtllm*",
172
    "--ignore-glob=docs/*",
173
    "--ignore-glob=components/src/dynamo/sglang/request_handlers/*",
174
    "--ignore-glob=components/src/dynamo/sglang/multimodal_utils/*",
175
    "--ignore-glob=components/src/dynamo/vllm/multimodal_utils/*",
176
    "--ignore-glob=examples/backends/sglang/slurm_jobs/*",
177
178
    # FIXME: Get relative/generic blob paths to work here
]
179
180
181
xfail_strict = true
log_cli_level = "INFO"
filterwarnings = [
182
    "error",
183
184
    # CUDA deprecation warnings from tensorrt_llm
    "ignore:.*cuda*:DeprecationWarning",
185
186
187
188
189
    # cuda.cudart/cuda.nvrtc deprecated in favor of cuda.bindings.* (cuda-python >=13)
    # Triggered by flashinfer 0.6.7+ during import of comm/mnnvl.py
    "ignore:The cuda\\..*module is deprecated:FutureWarning",
    # SGLang GGUF quantization emits UserWarning on non-CUDA platforms (arm64 CPU-only CI)
    "ignore:Only CUDA.*support GGUF quantization:UserWarning",
190
191
192
193
194
195
196
197
198
199
200
201
202
203
    # protobuf C extension warning
    "ignore:.*PyType_Spec.*custom tp_new.*:DeprecationWarning",
    # unclosed socket/event loop warnings
    "ignore:.*unclosed.*socket.*:ResourceWarning",
    "ignore:.*unclosed event loop.*:ResourceWarning",
    # unraisable exception warnings
    "ignore:.*Exception ignored in.*:pytest.PytestUnraisableExceptionWarning",
    # pynvml deprecation, temporary until upstream migrates to nvidia-ml-py
    "ignore:The pynvml package is deprecated.*:FutureWarning",
    # Dynamo's own KV events deprecation warning
    "ignore:Automatic KV events configuration is deprecated.*:FutureWarning",
    # Python 3.12 SWIG extension warning from third-party tokenizer deps
    "ignore:builtin type (SwigPyPacked|SwigPyObject|swigvarlink) has no __module__ attribute:DeprecationWarning",
    # Pydantic V2 deprecation warnings from TRTLLM dependencies
204
205
206
    "ignore:Support for class-based `config`.*:pydantic.warnings.PydanticDeprecatedSince20",
    "ignore:Using extra keyword arguments on `Field`.*:pydantic.warnings.PydanticDeprecatedSince20",
    "ignore:The `schema` method is deprecated.*:pydantic.warnings.PydanticDeprecatedSince20",
207
    # Pydantic field shadowing in tensorrt_llm.serve.openai_protocol.ResponseFormat
208
    'ignore:Field name "schema" in "ResponseFormat" shadows an attribute in parent:UserWarning',
209
    # pytest-benchmark automatically disables when xdist is active
210
    "ignore:.*Benchmarks are automatically disabled.*:pytest_benchmark.logger.PytestBenchmarkWarning",
211
    # torchao invalid escape sequences in docstrings at import time
212
    "ignore:.*invalid escape sequence.*:SyntaxWarning",
213
    # torchao import path changes (https://github.com/pytorch/ao/issues/2752)
214
    "ignore:Importing.*torchao\\.dtypes.*:DeprecationWarning",
215
216
217
218
    # Triton CPU fallback warning on CPU-only runners
    "ignore:Triton is not supported on current platform.*:UserWarning",
    # torch.jit.script_method deprecation from torch.utils.mkldnn
    "ignore:.*torch\\.jit\\.script_method.*is deprecated.*:DeprecationWarning",
219
220
    # torch.jit.script deprecation from modelopt.torch.quantization
    "ignore:`torch.jit.script` is deprecated:DeprecationWarning",
221
    # nvidia-modelopt warning about transformers version (transitive dep from TRT-LLM)
222
    "ignore:transformers version .* is incompatible with nvidia-modelopt.*:UserWarning",
223
224
225
    # SGLang quantization warnings on CPU-only runners
    "ignore:Only CUDA, HIP and XPU support AWQ currently.*:UserWarning",
    "ignore:Only CUDA support GGUF quantization currently.*:UserWarning",
226
]
227
228


229
230
# NOTE: Can also manually mark tests with @pytest.mark.asyncio
asyncio_mode = "auto"
Neelay Shah's avatar
Neelay Shah committed
231
markers = [
232
    "pre_merge: marks tests to run before merging",
233
    "post_merge: marks tests to run after merge",
234
    "parallel: marks tests that can run in parallel with pytest-xdist",
235
    "nightly: marks tests to run nightly",
236
    "skip_in_nightly: excludes a test from the nightly pipeline only (still runs in pre_merge/post_merge). Used via `not skip_in_nightly` in nightly-ci.yml marker filters",
237
    "frontend_api_surface_compliance: marks tests that validate Dynamo's HTTP API surface (Responses/Anthropic wire shape, tool-call routing) against upstream compliance harnesses",
238
    "weekly: marks tests to run weekly",
239
    "release: marks tests to run on release pipelines",
240
    "gpu_0: marks tests that don't require GPU",
Neelay Shah's avatar
Neelay Shah committed
241
242
    "gpu_1: marks tests to run on GPU",
    "gpu_2: marks tests to run on 2GPUs",
243
244
    "gpu_4: marks tests to run on 4GPUs",
    "gpu_8: marks tests to run on 8GPUs",
245
246
    "xpu_1: marks tests to run on XPU",
    "xpu_2: marks tests to run on 2XPUs",
247
    # These 5 (profiled_vram_gib and requested_*) are used for parallel pytest executions:
248
249
250
    "profiled_vram_gib(N): actual peak VRAM observed by nvidia-smi during profiling. Used for --max-vram-gib filtering and scheduler budget tracking",
    "requested_vllm_kv_cache_bytes(N): exact KV cache bytes for vLLM (skips memory profiling). Sets _PROFILE_PYTEST_KV_CACHE_BYTES. Most deterministic method for parallel execution",
    "requested_sglang_kv_tokens(N): max KV cache tokens for SGLang parallel execution. Sets _OVERRIDE_SGLANG_MAX_TOTAL_TOKENS to cap --max-total-tokens and prevent over-allocation",
251
252
    "requested_trtllm_kv_tokens(N): max KV cache tokens for TensorRT-LLM parallel execution. Sets _PROFILE_OVERRIDE_TRTLLM_MAX_TOTAL_TOKENS to cap KvCacheConfig.max_tokens via --override-engine-args",
    "requested_trtllm_vram_gib(N): max VRAM in GiB for TensorRT-LLM parallel execution. Sets _PROFILE_OVERRIDE_TRTLLM_MAX_GPU_TOTAL_BYTES to cap KvCacheConfig.max_gpu_total_bytes via --override-engine-args. Use for non-text workloads (video/image diffusion)",
Neelay Shah's avatar
Neelay Shah committed
253
254
255
256
    "e2e: marks tests as end-to-end tests",
    "integration: marks tests as integration tests",
    "unit: marks tests as unit tests",
    "stress: marks tests as stress tests",
257
    "performance: marks tests as performance tests",
258
    "benchmark: marks tests as benchmark tests",
259
    "none: marks tests that do not require a framework-specific runtime",
Neelay Shah's avatar
Neelay Shah committed
260
    "vllm: marks tests as requiring vllm",
261
    "trtllm: marks tests as requiring trtllm",
Neelay Shah's avatar
Neelay Shah committed
262
    "sglang: marks tests as requiring sglang",
263
    "lmcache: marks tests as requiring lmcache",
264
    "multimodal: marks tests as multimodal (image/video) tests",
Ryan Olson's avatar
Ryan Olson committed
265
    "slow: marks tests as known to be slow",
266
    "h100: marks tests to run on H100",
267
    "aiconfigurator: marks e2e tests that cover aiconfigurator functionality",
268
269
    "router: marks tests for router component",
    "planner: marks tests for planner component",
Alec's avatar
Alec committed
270
    "kvbm: marks tests for KV behavior and model determinism",
271
    "kvbm_concurrency: marks concurrency stress tests for KVBM (runs separately)",
272
    "model: model id used by a test or parameter",
273
274
    "custom_build: marks tests that require custom builds or special setup (e.g., MoE models)",
    "k8s: marks tests as requiring Kubernetes",
275
    "fault_tolerance: marks tests as fault tolerance tests",
276
    "deploy: marks tests as deployment tests",
277
278
    "framework_only: marks standard framework deployment tests (vllm, sglang, trtllm)",
    "framework_with_gaie: marks tests for GAIE (Gateway API Inference Extension) deployment",
279
280
281
282
283
284
285
    # Built-in markers
    "skip: skip this test",
    "skipif: skip if condition is true",
    "xfail: expected failure",
    "usefixtures: use fixtures",
    "parametrize: parameterized test",
    "filterwarnings: filter warnings",
286
287
288
    "asyncio: asyncio test marker",
    # Third-party plugin markers
    "timeout: test timeout in seconds (pytest-timeout plugin)",
Neelay Shah's avatar
Neelay Shah committed
289
]
290
291
292
293
294
295
296

# Linting/formatting
[tool.ruff]
# Same as Black.
line-length = 88
indent-width = 4

Blazej's avatar
Blazej committed
297
298
[tool.ruff.lint.extend-per-file-ignores]
"icp/tests/**/test_*.py" = ["F811", "F401"]
299
"*_inc.py" = ["F821"]
Blazej's avatar
Blazej committed
300

301
302
303
304
305
306
307
308
309
# This is IDE (e.g. Cursor's default Python language server)
# Configure it so that developers can use "go-to-definition", "hover types" and other
# features.
[tool.basedpyright]
extraPaths = ["components/src", "lib/bindings/python/src"]
# This is for external dependencies.
venvPath = "."
venv = ".venv"

310
[tool.mypy]
Neelay Shah's avatar
Neelay Shah committed
311

312
313
# --disable-error-code: WAR large set of errors due to mypy not being run
#   previously. We can slowly enable sets of errors to fix over time.
Neelay Shah's avatar
Neelay Shah committed
314
315
# disable_error_code = []

316
# --explicit-package-bases: WAR errors about duplicate module names used
317
318
319
#   throughout the llm examples. For example, the common module in
#   tensorrt_llm and vllm are both named common.
explicit_package_bases = true
Neelay Shah's avatar
Neelay Shah committed
320

jh-nv's avatar
jh-nv committed
321
322
323
324
325
check_untyped_defs = true

[[tool.mypy.overrides]]
# _version.py is generated at build time and does not exist in the source tree.
module = ["dynamo.*._version"]
Neelay Shah's avatar
Neelay Shah committed
326
327
ignore_missing_imports = true

jh-nv's avatar
jh-nv committed
328
329
330
331
[[tool.mypy.overrides]]
# Skip type checking for test files.
module = ["dynamo.*.tests.*", "dynamo.*.tests"]
ignore_errors = true
332
333
334
335
336

[[tool.mypy.overrides]]
# Manual planner helpers are operational scripts, not typed library surfaces.
module = ["dynamo.planner.manual.*"]
ignore_errors = true
jh-nv's avatar
jh-nv committed
337
338
339
340
341
342
343
344
345
346
347
348
349

[[tool.mypy.overrides]]
# Skip mypy analysis on backend framework internals.
# ignore_missing_imports silences import-not-found only when the backend
# is not installed (e.g. sglang/trtllm missing in the vllm container).
module = ["vllm", "vllm.*"]
follow_imports = "skip"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = ["sglang", "sglang.*"]
follow_imports = "skip"
ignore_missing_imports = true
350
351

[[tool.mypy.overrides]]
jh-nv's avatar
jh-nv committed
352
module = ["tensorrt_llm", "tensorrt_llm.*"]
353
follow_imports = "skip"
354
ignore_missing_imports = true
355

356
357
358
359
360
361
[[tool.mypy.overrides]]
# WAR mypy 1.18.x crash with numpy 1.26.x stubs:
# "Should never get here in normal mode, got TypeAlias:numpy.float64 instead of TypeInfo"
module = ["numpy", "numpy.*"]
follow_imports = "skip"

jh-nv's avatar
jh-nv committed
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
[[tool.mypy.overrides]]
# Third-party libs without type stubs or optional internal deps
# TODO: fix the ones that do have stub package
module = [
    "nvtx",
    "fsspec",
    "fsspec.*",
    "kubernetes",
    "kubernetes.*",
    "scipy",
    "scipy.*",
    "sklearn",
    "sklearn.*",
    "pandas",
    "pandas.*",
    "pmdarima",
    "pmdarima.*",
    "filterpy",
    "filterpy.*",
    "prophet",
    "prophet.*",
    "msgpack",
    "nixl",
    "nixl.*",
    "imageio",
    "imageio.*",
    "yaml",
    "prometheus_api_client",
    "prometheus_api_client.*",
    "aiohttp",
    "aiohttp.*",
    "vllm_omni",
    "vllm_omni.*",
    "modelexpress",
    "modelexpress.*",
    "kvbm",
    "kvbm.*",
    "diffusers",
    "diffusers.*",
    "PIL",
    "PIL.*",
    "torch",
    "torch.*",
    "transformers",
    "transformers.*",
    "cupy",
    "cupy.*",
    "gpu_memory_service",
    "gpu_memory_service.*",
    "pydantic",
    "pydantic.*",
    "uvloop",
    "prometheus_client",
    "prometheus_client.*",
    "pybase64",
    "blake3",
    "cupy_backends",
    "cupy_backends.*",
    "huggingface_hub",
    "huggingface_hub.*",
    "httpx",
    "httpx.*",
    "zmq",
    "zmq.*",
    "safetensors",
    "safetensors.*",
    "gradio",
    "gradio.*",
    "kubernetes_asyncio",
    "kubernetes_asyncio.*",
432
433
    "ray",
    "ray.*",
jh-nv's avatar
jh-nv committed
434
435
436
    "pydantic_core",
    "aiconfigurator",
    "aiconfigurator.*",
437
438
439
440
    "soundfile",
    "soundfile.*",
    "librosa",
    "librosa.*",
jh-nv's avatar
jh-nv committed
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
]
ignore_missing_imports = true

[[tool.mypy.overrides]]
# msgspec.Struct uses custom __init_subclass__ kwargs (frozen, gc) that mypy
# cannot resolve without the msgspec package installed.
module = ["msgspec", "msgspec.*"]
follow_imports = "skip"
ignore_missing_imports = true

[[tool.mypy.overrides]]
# Profiler module was never previously type-checked and has many
# union-attr / attr-defined issues. Skip errors for now.
module = ["dynamo.profiler.*"]
ignore_errors = true

457
458
[tool.sphinx]

459
# extra-content-head
460
461
462
463
464
465
466
467
468
469
470
471
extra_content_head = [
   '''
   <script src="https://assets.adobedtm.com/5d4962a43b79/c1061d2c5e7b/launch-191c2462b890.min.js" ></script>
   ''',
]

#extra-content-footer
extra_content_footer = [
   '''
   <script type="text/javascript">if (typeof _satellite !== "undefined") {_satellite.pageBottom();}</script>
   ''',
]