pyproject.toml 16.2 KB
Newer Older
1
# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
3
# SPDX-License-Identifier: Apache-2.0

Neelay Shah's avatar
Neelay Shah committed
4
[project]
5
name = "ai-dynamo"
6
version = "1.0.0"
Neelay Shah's avatar
Neelay Shah committed
7
8
9
description = "Distributed Inference Framework"
readme = "README.md"
authors = [
Neelay Shah's avatar
Neelay Shah committed
10
    { name = "NVIDIA Inc.", email = "sw-dl-dynamo@nvidia.com" },
Neelay Shah's avatar
Neelay Shah committed
11
]
12
13
license = { text = "Apache-2.0" }
license-files = ["LICENSE"]
Neelay Shah's avatar
Neelay Shah committed
14
15
requires-python = ">=3.10"
dependencies = [
16
    "ai-dynamo-runtime==1.0.0",
17
    "transformers>=4.56.0",
18
    "kubernetes>=32.0.1,<33.0.0",
19
    "prometheus_client>=0.23.1,<1.0",
20
21
    "msgspec>=0.19.0",
    "pyzmq>=26.0.0",
22
    "msgpack==1.1.2",
Neelay Shah's avatar
Neelay Shah committed
23
]
24

25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
classifiers = [
    "Development Status :: 4 - Beta",
    "Intended Audience :: Developers",
    "Intended Audience :: Science/Research",
    "Intended Audience :: Information Technology",
    "License :: OSI Approved :: Apache Software License",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Topic :: Scientific/Engineering",
    "Topic :: Scientific/Engineering :: Artificial Intelligence",
    "Operating System :: POSIX :: Linux",
]
keywords = ["llm", "genai", "inference", "nvidia", "distributed", "dynamo"]
Neelay Shah's avatar
Neelay Shah committed
40

41
42
43
[project.urls]
Repository = "https://github.com/ai-dynamo/dynamo.git"

44
[project.optional-dependencies]
45
46
trtllm =[
    "uvloop",
47
    "tensorrt-llm==1.3.0rc11",
48
49
]

50
51
vllm = [
    "uvloop",
52
    "nixl[cu12]<=0.10.1",
Alec's avatar
Alec committed
53
54
55
56
57
    "vllm[flashinfer,runai,otel]==0.19.0",
    # vllm-omni is installed separately in container builds (see
    # container/deps/vllm/install_vllm.sh). Do not add it to ai-dynamo[vllm]:
    # pip/uv dependency resolution for omni can override the vLLM torch stack.
    # "vllm-omni==...",
58
    "blake3>=1.0.0,<2.0.0",
59
60
    "soundfile>=0.13.1",
    "librosa>=0.10.0",
61
62
]

63
64
sglang = [
    "uvloop",
65
    "sglang[diffusion]==0.5.10.post1",
66
    "nixl[cu12]>=1.0.0,<1.1.0",
67
    "cupy-cuda12x>=13.0.0",
68
69
]

70
71
72
73
mocker = [
    "aiconfigurator>=0.7.0",
]

74
75
76
77
78
[project.entry-points.pytest11]
vllm_tests = "dynamo.vllm.tests.conftest"
trtllm_tests = "dynamo.trtllm.tests.conftest"
sglang_tests = "dynamo.sglang.tests.conftest"

79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
[dependency-groups]
docs = [
    # Core Sphinx
    "sphinx>=8.1",
    "nvidia-sphinx-theme>=0.0.8",
    # Sphinx extensions
    "ablog>=0.11",
    "sphinx-copybutton>=0.5",
    "sphinx-design>=0.6",
    "sphinx-prompt>=1.9",
    "sphinx-sitemap>=2.6",
    "sphinx-tabs>=3.4",
    "sphinx-book-theme>=1.1",
    "sphinxcontrib-mermaid>=1.0",
    "sphinxcontrib-bibtex>=2.6",
94
    "sphinx-reredirects>=1.0.0",
95
96
97
98
99
100
    # Markdown and notebook support
    "myst-parser>=4.0",
    "myst-nb>=1.2",
    "nbsphinx>=0.9",
]

101
[build-system]
102
103
104
requires = ["hatchling"]
build-backend = "hatchling.build"

105
106
107
[tool.hatch.build.hooks.custom]
path = "hatch_build.py"

108
[tool.hatch.build.targets.wheel]
109
packages = [
110
    "components/src/dynamo",
111
]
112

113
114
115
[tool.hatch.metadata]
allow-direct-references = true

116
117
118
119
120
[tool.codespell]
# note: pre-commit passes explicit lists of files here, which this skip file list doesn't override -
# this is only to allow you to run codespell interactively
# this also overrides the grpc_generated folder, since it is generated

121
122
# Ignore data files and auto-generated files
skip = "./.git,./.github,./lib/llm/tests/data,*.lock,*.sum"
123

124
125
# ignore allowed words used in code
ignore-words-list = "afterall,ser,ende"
126
127
# use the 'clear' dictionary for unambiguous spelling mistakes
builtin = "clear"
128
129
# use custom dictionary in addition to the built-in one
dictionary = "./codespell.txt"
130
131
132
133
134
135
136
137
138
139
140
141
142
143
# disable warnings about binary files and wrong encoding
quiet-level = 3

[tool.isort]
profile = "black"
use_parentheses = true
multi_line_output = 3
include_trailing_comma = true
force_grid_wrap = 0
ensure_newline_before_comments = true
line_length = 88
balanced_wrapping = true
indent = "    "
skip = ["build"]
144
known_first_party = ["dynamo", "deploy"]
145
146
147
148
149
150
# isort may confuse what is 1st or 3rd library. e.g.
# when dynamo/vllm/omni/xx.py import vllm, local isort may treat this `vllm` as first
# party heuristically. This causes local sort differs from GitHub sort and pre-commit
# failure. To mitigate 1) one can install 3rd party lib so that isort is aware of it,
# 2) hardcode 3rd party lib here, 3) add "# isort: skip_file" to problematic files
# as the last resort.
151
known_third_party = ["vllm", "tensorrt_llm", "sglang", "aiconfigurator"]
152
153
154

[tool.pytest.ini_options]
minversion = "8.0"
155
tmp_path_retention_policy = "failed"
Neelay Shah's avatar
Neelay Shah committed
156
157

# NOTE
jh-nv's avatar
jh-nv committed
158
159
# Keep these ignores in pytest collection to avoid duplicate-module collection
# errors (for example, backend trees that include multiple model.py files).
160
161
162
163
164
165
addopts = [
    "-ra",
    "--showlocals",
    "--strict-markers",
    "--strict-config",
    "--ignore-glob=*model.py",
Ryan Olson's avatar
Ryan Olson committed
166
    "--ignore-glob=*vllm_integration*",
167
    "--ignore-glob=*trtllm_integration*",
Richard Huo's avatar
Richard Huo committed
168
    "--ignore-glob=*kvbm/python/kvbm*",
169
    "--ignore-glob=*_inc.py",
170
    "--ignore-glob=*/llm/tensorrtllm*",
171
    "--ignore-glob=docs/*",
172
    "--ignore-glob=components/src/dynamo/sglang/request_handlers/*",
173
    "--ignore-glob=components/src/dynamo/sglang/multimodal_utils/*",
174
    "--ignore-glob=components/src/dynamo/vllm/multimodal_utils/*",
175
    "--ignore-glob=examples/backends/sglang/slurm_jobs/*",
176
177
    # FIXME: Get relative/generic blob paths to work here
]
178
179
180
xfail_strict = true
log_cli_level = "INFO"
filterwarnings = [
181
    "error",
182
183
    # CUDA deprecation warnings from tensorrt_llm
    "ignore:.*cuda*:DeprecationWarning",
184
185
186
187
188
    # cuda.cudart/cuda.nvrtc deprecated in favor of cuda.bindings.* (cuda-python >=13)
    # Triggered by flashinfer 0.6.7+ during import of comm/mnnvl.py
    "ignore:The cuda\\..*module is deprecated:FutureWarning",
    # SGLang GGUF quantization emits UserWarning on non-CUDA platforms (arm64 CPU-only CI)
    "ignore:Only CUDA.*support GGUF quantization:UserWarning",
189
190
191
192
193
194
195
196
197
198
199
200
201
202
    # protobuf C extension warning
    "ignore:.*PyType_Spec.*custom tp_new.*:DeprecationWarning",
    # unclosed socket/event loop warnings
    "ignore:.*unclosed.*socket.*:ResourceWarning",
    "ignore:.*unclosed event loop.*:ResourceWarning",
    # unraisable exception warnings
    "ignore:.*Exception ignored in.*:pytest.PytestUnraisableExceptionWarning",
    # pynvml deprecation, temporary until upstream migrates to nvidia-ml-py
    "ignore:The pynvml package is deprecated.*:FutureWarning",
    # Dynamo's own KV events deprecation warning
    "ignore:Automatic KV events configuration is deprecated.*:FutureWarning",
    # Python 3.12 SWIG extension warning from third-party tokenizer deps
    "ignore:builtin type (SwigPyPacked|SwigPyObject|swigvarlink) has no __module__ attribute:DeprecationWarning",
    # Pydantic V2 deprecation warnings from TRTLLM dependencies
203
204
205
    "ignore:Support for class-based `config`.*:pydantic.warnings.PydanticDeprecatedSince20",
    "ignore:Using extra keyword arguments on `Field`.*:pydantic.warnings.PydanticDeprecatedSince20",
    "ignore:The `schema` method is deprecated.*:pydantic.warnings.PydanticDeprecatedSince20",
206
    # Pydantic field shadowing in tensorrt_llm.serve.openai_protocol.ResponseFormat
207
    'ignore:Field name "schema" in "ResponseFormat" shadows an attribute in parent:UserWarning',
208
    # pytest-benchmark automatically disables when xdist is active
209
    "ignore:.*Benchmarks are automatically disabled.*:pytest_benchmark.logger.PytestBenchmarkWarning",
210
    # torchao invalid escape sequences in docstrings at import time
211
    "ignore:.*invalid escape sequence.*:SyntaxWarning",
212
    # torchao import path changes (https://github.com/pytorch/ao/issues/2752)
213
    "ignore:Importing.*torchao\\.dtypes.*:DeprecationWarning",
214
215
216
217
    # Triton CPU fallback warning on CPU-only runners
    "ignore:Triton is not supported on current platform.*:UserWarning",
    # torch.jit.script_method deprecation from torch.utils.mkldnn
    "ignore:.*torch\\.jit\\.script_method.*is deprecated.*:DeprecationWarning",
218
219
    # torch.jit.script deprecation from modelopt.torch.quantization
    "ignore:`torch.jit.script` is deprecated:DeprecationWarning",
220
    # nvidia-modelopt warning about transformers version (transitive dep from TRT-LLM)
221
    "ignore:transformers version .* is incompatible with nvidia-modelopt.*:UserWarning",
222
223
224
    # SGLang quantization warnings on CPU-only runners
    "ignore:Only CUDA, HIP and XPU support AWQ currently.*:UserWarning",
    "ignore:Only CUDA support GGUF quantization currently.*:UserWarning",
225
]
226
227


228
229
# NOTE: Can also manually mark tests with @pytest.mark.asyncio
asyncio_mode = "auto"
Neelay Shah's avatar
Neelay Shah committed
230
markers = [
231
    "pre_merge: marks tests to run before merging",
232
    "post_merge: marks tests to run after merge",
233
    "parallel: marks tests that can run in parallel with pytest-xdist",
234
235
    "nightly: marks tests to run nightly",
    "weekly: marks tests to run weekly",
236
    "release: marks tests to run on release pipelines",
237
    "gpu_0: marks tests that don't require GPU",
Neelay Shah's avatar
Neelay Shah committed
238
239
    "gpu_1: marks tests to run on GPU",
    "gpu_2: marks tests to run on 2GPUs",
240
241
    "gpu_4: marks tests to run on 4GPUs",
    "gpu_8: marks tests to run on 8GPUs",
242
243
    "xpu_1: marks tests to run on XPU",
    "xpu_2: marks tests to run on 2XPUs",
244
    # These 5 (profiled_vram_gib and requested_*) are used for parallel pytest executions:
245
246
247
    "profiled_vram_gib(N): actual peak VRAM observed by nvidia-smi during profiling. Used for --max-vram-gib filtering and scheduler budget tracking",
    "requested_vllm_kv_cache_bytes(N): exact KV cache bytes for vLLM (skips memory profiling). Sets _PROFILE_PYTEST_KV_CACHE_BYTES. Most deterministic method for parallel execution",
    "requested_sglang_kv_tokens(N): max KV cache tokens for SGLang parallel execution. Sets _OVERRIDE_SGLANG_MAX_TOTAL_TOKENS to cap --max-total-tokens and prevent over-allocation",
248
249
    "requested_trtllm_kv_tokens(N): max KV cache tokens for TensorRT-LLM parallel execution. Sets _PROFILE_OVERRIDE_TRTLLM_MAX_TOTAL_TOKENS to cap KvCacheConfig.max_tokens via --override-engine-args",
    "requested_trtllm_vram_gib(N): max VRAM in GiB for TensorRT-LLM parallel execution. Sets _PROFILE_OVERRIDE_TRTLLM_MAX_GPU_TOTAL_BYTES to cap KvCacheConfig.max_gpu_total_bytes via --override-engine-args. Use for non-text workloads (video/image diffusion)",
Neelay Shah's avatar
Neelay Shah committed
250
251
252
253
    "e2e: marks tests as end-to-end tests",
    "integration: marks tests as integration tests",
    "unit: marks tests as unit tests",
    "stress: marks tests as stress tests",
254
    "performance: marks tests as performance tests",
255
    "benchmark: marks tests as benchmark tests",
256
    "none: marks tests that do not require a framework-specific runtime",
Neelay Shah's avatar
Neelay Shah committed
257
    "vllm: marks tests as requiring vllm",
258
    "trtllm: marks tests as requiring trtllm",
Neelay Shah's avatar
Neelay Shah committed
259
    "sglang: marks tests as requiring sglang",
260
    "lmcache: marks tests as requiring lmcache",
261
    "multimodal: marks tests as multimodal (image/video) tests",
Ryan Olson's avatar
Ryan Olson committed
262
    "slow: marks tests as known to be slow",
263
    "h100: marks tests to run on H100",
264
    "aiconfigurator: marks e2e tests that cover aiconfigurator functionality",
265
266
    "router: marks tests for router component",
    "planner: marks tests for planner component",
Alec's avatar
Alec committed
267
    "kvbm: marks tests for KV behavior and model determinism",
268
    "kvbm_concurrency: marks concurrency stress tests for KVBM (runs separately)",
269
    "model: model id used by a test or parameter",
270
271
    "custom_build: marks tests that require custom builds or special setup (e.g., MoE models)",
    "k8s: marks tests as requiring Kubernetes",
272
    "fault_tolerance: marks tests as fault tolerance tests",
273
    "deploy: marks tests as deployment tests",
274
275
    "framework_only: marks standard framework deployment tests (vllm, sglang, trtllm)",
    "framework_with_gaie: marks tests for GAIE (Gateway API Inference Extension) deployment",
276
277
278
279
280
281
282
    # Built-in markers
    "skip: skip this test",
    "skipif: skip if condition is true",
    "xfail: expected failure",
    "usefixtures: use fixtures",
    "parametrize: parameterized test",
    "filterwarnings: filter warnings",
283
284
285
    "asyncio: asyncio test marker",
    # Third-party plugin markers
    "timeout: test timeout in seconds (pytest-timeout plugin)",
Neelay Shah's avatar
Neelay Shah committed
286
]
287
288
289
290
291
292
293

# Linting/formatting
[tool.ruff]
# Same as Black.
line-length = 88
indent-width = 4

Blazej's avatar
Blazej committed
294
295
[tool.ruff.lint.extend-per-file-ignores]
"icp/tests/**/test_*.py" = ["F811", "F401"]
296
"*_inc.py" = ["F821"]
Blazej's avatar
Blazej committed
297

298
299
300
301
302
303
304
305
306
# This is IDE (e.g. Cursor's default Python language server)
# Configure it so that developers can use "go-to-definition", "hover types" and other
# features.
[tool.basedpyright]
extraPaths = ["components/src", "lib/bindings/python/src"]
# This is for external dependencies.
venvPath = "."
venv = ".venv"

307
[tool.mypy]
Neelay Shah's avatar
Neelay Shah committed
308

309
310
# --disable-error-code: WAR large set of errors due to mypy not being run
#   previously. We can slowly enable sets of errors to fix over time.
Neelay Shah's avatar
Neelay Shah committed
311
312
# disable_error_code = []

313
# --explicit-package-bases: WAR errors about duplicate module names used
314
315
316
#   throughout the llm examples. For example, the common module in
#   tensorrt_llm and vllm are both named common.
explicit_package_bases = true
Neelay Shah's avatar
Neelay Shah committed
317

jh-nv's avatar
jh-nv committed
318
319
320
321
322
check_untyped_defs = true

[[tool.mypy.overrides]]
# _version.py is generated at build time and does not exist in the source tree.
module = ["dynamo.*._version"]
Neelay Shah's avatar
Neelay Shah committed
323
324
ignore_missing_imports = true

jh-nv's avatar
jh-nv committed
325
326
327
328
[[tool.mypy.overrides]]
# Skip type checking for test files.
module = ["dynamo.*.tests.*", "dynamo.*.tests"]
ignore_errors = true
329
330
331
332
333

[[tool.mypy.overrides]]
# Manual planner helpers are operational scripts, not typed library surfaces.
module = ["dynamo.planner.manual.*"]
ignore_errors = true
jh-nv's avatar
jh-nv committed
334
335
336
337
338
339
340
341
342
343
344
345
346

[[tool.mypy.overrides]]
# Skip mypy analysis on backend framework internals.
# ignore_missing_imports silences import-not-found only when the backend
# is not installed (e.g. sglang/trtllm missing in the vllm container).
module = ["vllm", "vllm.*"]
follow_imports = "skip"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = ["sglang", "sglang.*"]
follow_imports = "skip"
ignore_missing_imports = true
347
348

[[tool.mypy.overrides]]
jh-nv's avatar
jh-nv committed
349
module = ["tensorrt_llm", "tensorrt_llm.*"]
350
follow_imports = "skip"
351
ignore_missing_imports = true
352

353
354
355
356
357
358
[[tool.mypy.overrides]]
# WAR mypy 1.18.x crash with numpy 1.26.x stubs:
# "Should never get here in normal mode, got TypeAlias:numpy.float64 instead of TypeInfo"
module = ["numpy", "numpy.*"]
follow_imports = "skip"

jh-nv's avatar
jh-nv committed
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
[[tool.mypy.overrides]]
# Third-party libs without type stubs or optional internal deps
# TODO: fix the ones that do have stub package
module = [
    "nvtx",
    "fsspec",
    "fsspec.*",
    "kubernetes",
    "kubernetes.*",
    "scipy",
    "scipy.*",
    "sklearn",
    "sklearn.*",
    "pandas",
    "pandas.*",
    "pmdarima",
    "pmdarima.*",
    "filterpy",
    "filterpy.*",
    "prophet",
    "prophet.*",
    "msgpack",
    "nixl",
    "nixl.*",
    "imageio",
    "imageio.*",
    "yaml",
    "prometheus_api_client",
    "prometheus_api_client.*",
    "aiohttp",
    "aiohttp.*",
    "vllm_omni",
    "vllm_omni.*",
    "modelexpress",
    "modelexpress.*",
    "kvbm",
    "kvbm.*",
    "diffusers",
    "diffusers.*",
    "PIL",
    "PIL.*",
    "torch",
    "torch.*",
    "transformers",
    "transformers.*",
    "cupy",
    "cupy.*",
    "gpu_memory_service",
    "gpu_memory_service.*",
    "pydantic",
    "pydantic.*",
    "uvloop",
    "prometheus_client",
    "prometheus_client.*",
    "pybase64",
    "blake3",
    "cupy_backends",
    "cupy_backends.*",
    "huggingface_hub",
    "huggingface_hub.*",
    "httpx",
    "httpx.*",
    "zmq",
    "zmq.*",
    "safetensors",
    "safetensors.*",
    "gradio",
    "gradio.*",
    "kubernetes_asyncio",
    "kubernetes_asyncio.*",
429
430
    "ray",
    "ray.*",
jh-nv's avatar
jh-nv committed
431
432
433
    "pydantic_core",
    "aiconfigurator",
    "aiconfigurator.*",
434
435
436
437
    "soundfile",
    "soundfile.*",
    "librosa",
    "librosa.*",
jh-nv's avatar
jh-nv committed
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
]
ignore_missing_imports = true

[[tool.mypy.overrides]]
# msgspec.Struct uses custom __init_subclass__ kwargs (frozen, gc) that mypy
# cannot resolve without the msgspec package installed.
module = ["msgspec", "msgspec.*"]
follow_imports = "skip"
ignore_missing_imports = true

[[tool.mypy.overrides]]
# Profiler module was never previously type-checked and has many
# union-attr / attr-defined issues. Skip errors for now.
module = ["dynamo.profiler.*"]
ignore_errors = true

454
455
[tool.sphinx]

456
# extra-content-head
457
458
459
460
461
462
463
464
465
466
467
468
extra_content_head = [
   '''
   <script src="https://assets.adobedtm.com/5d4962a43b79/c1061d2c5e7b/launch-191c2462b890.min.js" ></script>
   ''',
]

#extra-content-footer
extra_content_footer = [
   '''
   <script type="text/javascript">if (typeof _satellite !== "undefined") {_satellite.pageBottom();}</script>
   ''',
]