Merge tag 'v0.7.2' into v0.7.2-dev

benchmarks/benchmark_latency.py

+# SPDX-License-Identifier: Apache-2.0
 """Benchmark the latency of processing a single batch of requests."""
 import argparse
 import dataclasses

benchmarks/benchmark_long_document_qa_throughput.py

+# SPDX-License-Identifier: Apache-2.0
 """
 Offline benchmark to test the long document QA throughput.
 

benchmarks/benchmark_prefix_caching.py

+# SPDX-License-Identifier: Apache-2.0
 """
 Benchmark the efficiency of prefix caching.
 

benchmarks/benchmark_prioritization.py

+# SPDX-License-Identifier: Apache-2.0
 """Benchmark offline prioritization."""
 import argparse
 import dataclasses

benchmarks/benchmark_serving.py

+# SPDX-License-Identifier: Apache-2.0
 r"""Benchmark online serving throughput.
 
 On the server side, run one of the following commands:

benchmarks/benchmark_serving_guided.py

+# SPDX-License-Identifier: Apache-2.0
 r"""Benchmark online serving throughput with guided decoding.
 
 On the server side, run one of the following commands:

benchmarks/benchmark_throughput.py

+# SPDX-License-Identifier: Apache-2.0
 """Benchmark offline inference throughput."""
 import argparse
 import dataclasses

benchmarks/cutlass_benchmarks/sparse_benchmarks.py

+# SPDX-License-Identifier: Apache-2.0
+
 import argparse
 import copy
 import itertools

benchmarks/cutlass_benchmarks/utils.py

+# SPDX-License-Identifier: Apache-2.0
+
 # Cutlass bench utils
 from typing import Iterable, Tuple
 

benchmarks/cutlass_benchmarks/w8a8_benchmarks.py

+# SPDX-License-Identifier: Apache-2.0
+
 import argparse
 import copy
 import itertools

benchmarks/cutlass_benchmarks/weight_shapes.py

+# SPDX-License-Identifier: Apache-2.0
+
 # Weight Shapes are in the format
 # ([K, N], TP_SPLIT_DIM)
 # Example:

benchmarks/disagg_benchmarks/disagg_prefill_proxy_server.py

+# SPDX-License-Identifier: Apache-2.0
+
 import os
 
 import aiohttp

benchmarks/disagg_benchmarks/round_robin_proxy.py

+# SPDX-License-Identifier: Apache-2.0
+
 import asyncio
 import itertools
 

benchmarks/disagg_benchmarks/visualize_benchmark_results.py

+# SPDX-License-Identifier: Apache-2.0
+
 import json
 
 import matplotlib.pyplot as plt

benchmarks/fused_kernels/layernorm_rms_benchmarks.py

+# SPDX-License-Identifier: Apache-2.0
+
 import pickle as pkl
 import time
 from dataclasses import dataclass

benchmarks/kernels/benchmark_aqlm.py

+# SPDX-License-Identifier: Apache-2.0
+
 import os
 import sys
 from typing import Optional

benchmarks/kernels/benchmark_layernorm.py

+# SPDX-License-Identifier: Apache-2.0
+
 import time
 
 import torch

benchmarks/kernels/benchmark_lora.py

+# SPDX-License-Identifier: Apache-2.0
+
 import argparse
 import copy
 import json

benchmarks/kernels/benchmark_machete.py

+# SPDX-License-Identifier: Apache-2.0
+
 import argparse
 import copy
 import itertools

benchmarks/kernels/benchmark_marlin.py

+# SPDX-License-Identifier: Apache-2.0
+
 from typing import List
 
 import torch