Skip to content

GitLab

Commit 7c4f76e3 authored by zhuwenwen's avatar zhuwenwen
Browse files

merge v0.4.0

parents 2da0dd3e 51c31bc1
Hide whitespace changes
Inline Side-by-side
Showing with 528 additions and 504 deletions
examples/offline_inference_with_prefix.py
View file @ 7c4f76e3
...@@ -22,7 +22,7 @@ prompts = [ ...@@ -22,7 +22,7 @@ prompts = [
sampling_params = SamplingParams(temperature=0.0) sampling_params = SamplingParams(temperature=0.0)
# Create an LLM. # Create an LLM.
llm = LLM(model="facebook/opt-125m") llm = LLM(model="facebook/opt-125m", enable_prefix_caching=True)
generating_prompts = [prefix + prompt for prompt in prompts] generating_prompts = [prefix + prompt for prompt in prompts]
...@@ -37,20 +37,14 @@ for output in outputs: ...@@ -37,20 +37,14 @@ for output in outputs:
print("-" * 80) print("-" * 80)
# -1 since the last token can change when concatenating prompts. # The llm.generate call will batch all prompts and send the batch at once
prefix_pos = len(llm.llm_engine.tokenizer.encode(prefix)) - 1 # if resources allow. The prefix will only be cached after the first batch
# is processed, so we need to call generate once to calculate the prefix
# The llm.generate call will batch all prompts and send the batch at once if resources allow. # and cache it.
# The prefix will only be cached after the first batch is processed, so we need to call generate once outputs = llm.generate(generating_prompts[0], sampling_params)
# to calculate the prefix and cache it.
outputs = llm.generate(generating_prompts[0],
sampling_params,
prefix_pos=[prefix_pos])
# Subsequent batches can leverage the cached prefix # Subsequent batches can leverage the cached prefix
outputs = llm.generate(generating_prompts, outputs = llm.generate(generating_prompts, sampling_params)
sampling_params,
prefix_pos=[prefix_pos] * len(generating_prompts))
# Print the outputs. You should see the same outputs as before # Print the outputs. You should see the same outputs as before
for output in outputs: for output in outputs:
......
examples/production_monitoring/grafana.json
View file @ 7c4f76e3
{ {
"__inputs": [
{
"name": "DS_PROMETHEUS",
"label": "prometheus",
"description": "",
"type": "datasource",
"pluginId": "prometheus",
"pluginName": "Prometheus"
}
],
"__elements": {},
"__requires": [
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "10.2.3"
},
{
"type": "datasource",
"id": "prometheus",
"name": "Prometheus",
"version": "1.0.0"
},
{
"type": "panel",
"id": "timeseries",
"name": "Time series",
"version": ""
}
],
"annotations": { "annotations": {
"list": [ "list": [
{ {
...@@ -42,6 +11,12 @@ ...@@ -42,6 +11,12 @@
"hide": true, "hide": true,
"iconColor": "rgba(0, 211, 255, 1)", "iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts", "name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard" "type": "dashboard"
} }
] ]
...@@ -50,14 +25,14 @@ ...@@ -50,14 +25,14 @@
"editable": true, "editable": true,
"fiscalYearStartMonth": 0, "fiscalYearStartMonth": 0,
"graphTooltip": 0, "graphTooltip": 0,
"id": null, "id": 29,
"links": [], "links": [],
"liveNow": false, "liveNow": false,
"panels": [ "panels": [
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "prometheus"
}, },
"description": "End to end request latency measured in seconds.", "description": "End to end request latency measured in seconds.",
"fieldConfig": { "fieldConfig": {
...@@ -66,7 +41,6 @@ ...@@ -66,7 +41,6 @@
"mode": "palette-classic" "mode": "palette-classic"
}, },
"custom": { "custom": {
"axisBorderShow": false,
"axisCenteredZero": false, "axisCenteredZero": false,
"axisColorMode": "text", "axisColorMode": "text",
"axisLabel": "", "axisLabel": "",
...@@ -80,7 +54,6 @@ ...@@ -80,7 +54,6 @@
"tooltip": false, "tooltip": false,
"viz": false "viz": false
}, },
"insertNulls": false,
"lineInterpolation": "linear", "lineInterpolation": "linear",
"lineWidth": 1, "lineWidth": 1,
"pointSize": 5, "pointSize": 5,
...@@ -138,11 +111,11 @@ ...@@ -138,11 +111,11 @@
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "prometheus"
}, },
"disableTextWrap": false, "disableTextWrap": false,
"editorMode": "builder", "editorMode": "builder",
"expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket[$__rate_interval])))", "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
"fullMetaSearch": false, "fullMetaSearch": false,
"includeNullMetadata": false, "includeNullMetadata": false,
"instant": false, "instant": false,
...@@ -154,11 +127,11 @@ ...@@ -154,11 +127,11 @@
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "prometheus"
}, },
"disableTextWrap": false, "disableTextWrap": false,
"editorMode": "builder", "editorMode": "builder",
"expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket[$__rate_interval])))", "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
"fullMetaSearch": false, "fullMetaSearch": false,
"hide": false, "hide": false,
"includeNullMetadata": false, "includeNullMetadata": false,
...@@ -171,11 +144,11 @@ ...@@ -171,11 +144,11 @@
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "prometheus"
}, },
"disableTextWrap": false, "disableTextWrap": false,
"editorMode": "builder", "editorMode": "builder",
"expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket[$__rate_interval])))", "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
"fullMetaSearch": false, "fullMetaSearch": false,
"hide": false, "hide": false,
"includeNullMetadata": false, "includeNullMetadata": false,
...@@ -188,11 +161,11 @@ ...@@ -188,11 +161,11 @@
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "prometheus"
}, },
"disableTextWrap": false, "disableTextWrap": false,
"editorMode": "builder", "editorMode": "builder",
"expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket[$__rate_interval])))", "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
"fullMetaSearch": false, "fullMetaSearch": false,
"hide": false, "hide": false,
"includeNullMetadata": false, "includeNullMetadata": false,
...@@ -205,10 +178,10 @@ ...@@ -205,10 +178,10 @@
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "prometheus"
}, },
"editorMode": "code", "editorMode": "code",
"expr": "rate(vllm:e2e_request_latency_seconds_sum[$__rate_interval])\n/\nrate(vllm:e2e_request_latency_seconds_count[$__rate_interval])", "expr": "rate(vllm:e2e_request_latency_seconds_sum{model_name=\"$model_name\"}[$__rate_interval])\n/\nrate(vllm:e2e_request_latency_seconds_count{model_name=\"$model_name\"}[$__rate_interval])",
"hide": false, "hide": false,
"instant": false, "instant": false,
"legendFormat": "Average", "legendFormat": "Average",
...@@ -222,7 +195,7 @@ ...@@ -222,7 +195,7 @@
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "prometheus"
}, },
"description": "Number of tokens processed per second", "description": "Number of tokens processed per second",
"fieldConfig": { "fieldConfig": {
...@@ -231,7 +204,6 @@ ...@@ -231,7 +204,6 @@
"mode": "palette-classic" "mode": "palette-classic"
}, },
"custom": { "custom": {
"axisBorderShow": false,
"axisCenteredZero": false, "axisCenteredZero": false,
"axisColorMode": "text", "axisColorMode": "text",
"axisLabel": "", "axisLabel": "",
...@@ -245,7 +217,6 @@ ...@@ -245,7 +217,6 @@
"tooltip": false, "tooltip": false,
"viz": false "viz": false
}, },
"insertNulls": false,
"lineInterpolation": "linear", "lineInterpolation": "linear",
"lineWidth": 1, "lineWidth": 1,
"pointSize": 5, "pointSize": 5,
...@@ -302,11 +273,11 @@ ...@@ -302,11 +273,11 @@
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "prometheus"
}, },
"disableTextWrap": false, "disableTextWrap": false,
"editorMode": "builder", "editorMode": "builder",
"expr": "rate(vllm:prompt_tokens_total[$__rate_interval])", "expr": "rate(vllm:prompt_tokens_total{model_name=\"$model_name\"}[$__rate_interval])",
"fullMetaSearch": false, "fullMetaSearch": false,
"includeNullMetadata": false, "includeNullMetadata": false,
"instant": false, "instant": false,
...@@ -318,11 +289,11 @@ ...@@ -318,11 +289,11 @@
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "prometheus"
}, },
"disableTextWrap": false, "disableTextWrap": false,
"editorMode": "builder", "editorMode": "builder",
"expr": "rate(vllm:generation_tokens_total[$__rate_interval])", "expr": "rate(vllm:generation_tokens_total{model_name=\"$model_name\"}[$__rate_interval])",
"fullMetaSearch": false, "fullMetaSearch": false,
"hide": false, "hide": false,
"includeNullMetadata": false, "includeNullMetadata": false,
...@@ -339,7 +310,7 @@ ...@@ -339,7 +310,7 @@
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "prometheus"
}, },
"description": "Inter token latency in seconds.", "description": "Inter token latency in seconds.",
"fieldConfig": { "fieldConfig": {
...@@ -348,7 +319,6 @@ ...@@ -348,7 +319,6 @@
"mode": "palette-classic" "mode": "palette-classic"
}, },
"custom": { "custom": {
"axisBorderShow": false,
"axisCenteredZero": false, "axisCenteredZero": false,
"axisColorMode": "text", "axisColorMode": "text",
"axisLabel": "", "axisLabel": "",
...@@ -362,7 +332,6 @@ ...@@ -362,7 +332,6 @@
"tooltip": false, "tooltip": false,
"viz": false "viz": false
}, },
"insertNulls": false,
"lineInterpolation": "linear", "lineInterpolation": "linear",
"lineWidth": 1, "lineWidth": 1,
"pointSize": 5, "pointSize": 5,
...@@ -420,11 +389,11 @@ ...@@ -420,11 +389,11 @@
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "prometheus"
}, },
"disableTextWrap": false, "disableTextWrap": false,
"editorMode": "builder", "editorMode": "builder",
"expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket[$__rate_interval])))", "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
"fullMetaSearch": false, "fullMetaSearch": false,
"includeNullMetadata": false, "includeNullMetadata": false,
"instant": false, "instant": false,
...@@ -436,11 +405,11 @@ ...@@ -436,11 +405,11 @@
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "prometheus"
}, },
"disableTextWrap": false, "disableTextWrap": false,
"editorMode": "builder", "editorMode": "builder",
"expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket[$__rate_interval])))", "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
"fullMetaSearch": false, "fullMetaSearch": false,
"hide": false, "hide": false,
"includeNullMetadata": false, "includeNullMetadata": false,
...@@ -453,11 +422,11 @@ ...@@ -453,11 +422,11 @@
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "prometheus"
}, },
"disableTextWrap": false, "disableTextWrap": false,
"editorMode": "builder", "editorMode": "builder",
"expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket[$__rate_interval])))", "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
"fullMetaSearch": false, "fullMetaSearch": false,
"hide": false, "hide": false,
"includeNullMetadata": false, "includeNullMetadata": false,
...@@ -470,11 +439,11 @@ ...@@ -470,11 +439,11 @@
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "prometheus"
}, },
"disableTextWrap": false, "disableTextWrap": false,
"editorMode": "builder", "editorMode": "builder",
"expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket[$__rate_interval])))", "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
"fullMetaSearch": false, "fullMetaSearch": false,
"hide": false, "hide": false,
"includeNullMetadata": false, "includeNullMetadata": false,
...@@ -487,10 +456,10 @@ ...@@ -487,10 +456,10 @@
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "prometheus"
}, },
"editorMode": "code", "editorMode": "code",
"expr": "rate(vllm:time_per_output_token_seconds_sum[$__rate_interval])\n/\nrate(vllm:time_per_output_token_seconds_count[$__rate_interval])", "expr": "rate(vllm:time_per_output_token_seconds_sum{model_name=\"$model_name\"}[$__rate_interval])\n/\nrate(vllm:time_per_output_token_seconds_count{model_name=\"$model_name\"}[$__rate_interval])",
"hide": false, "hide": false,
"instant": false, "instant": false,
"legendFormat": "Mean", "legendFormat": "Mean",
...@@ -504,7 +473,7 @@ ...@@ -504,7 +473,7 @@
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "prometheus"
}, },
"description": "Number of requests in RUNNING, WAITING, and SWAPPED state", "description": "Number of requests in RUNNING, WAITING, and SWAPPED state",
"fieldConfig": { "fieldConfig": {
...@@ -513,7 +482,6 @@ ...@@ -513,7 +482,6 @@
"mode": "palette-classic" "mode": "palette-classic"
}, },
"custom": { "custom": {
"axisBorderShow": false,
"axisCenteredZero": false, "axisCenteredZero": false,
"axisColorMode": "text", "axisColorMode": "text",
"axisLabel": "", "axisLabel": "",
...@@ -527,7 +495,6 @@ ...@@ -527,7 +495,6 @@
"tooltip": false, "tooltip": false,
"viz": false "viz": false
}, },
"insertNulls": false,
"lineInterpolation": "linear", "lineInterpolation": "linear",
"lineWidth": 1, "lineWidth": 1,
"pointSize": 5, "pointSize": 5,
...@@ -585,11 +552,11 @@ ...@@ -585,11 +552,11 @@
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "prometheus"
}, },
"disableTextWrap": false, "disableTextWrap": false,
"editorMode": "builder", "editorMode": "builder",
"expr": "vllm:num_requests_running", "expr": "vllm:num_requests_running{model_name=\"$model_name\"}",
"fullMetaSearch": false, "fullMetaSearch": false,
"includeNullMetadata": true, "includeNullMetadata": true,
"instant": false, "instant": false,
...@@ -601,11 +568,11 @@ ...@@ -601,11 +568,11 @@
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "prometheus"
}, },
"disableTextWrap": false, "disableTextWrap": false,
"editorMode": "builder", "editorMode": "builder",
"expr": "vllm:num_requests_swapped", "expr": "vllm:num_requests_swapped{model_name=\"$model_name\"}",
"fullMetaSearch": false, "fullMetaSearch": false,
"hide": false, "hide": false,
"includeNullMetadata": true, "includeNullMetadata": true,
...@@ -618,11 +585,11 @@ ...@@ -618,11 +585,11 @@
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "prometheus"
}, },
"disableTextWrap": false, "disableTextWrap": false,
"editorMode": "builder", "editorMode": "builder",
"expr": "vllm:num_requests_waiting", "expr": "vllm:num_requests_waiting{model_name=\"$model_name\"}",
"fullMetaSearch": false, "fullMetaSearch": false,
"hide": false, "hide": false,
"includeNullMetadata": true, "includeNullMetadata": true,
...@@ -639,7 +606,7 @@ ...@@ -639,7 +606,7 @@
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "prometheus"
}, },
"description": "P50, P90, P95, and P99 TTFT latency in seconds.", "description": "P50, P90, P95, and P99 TTFT latency in seconds.",
"fieldConfig": { "fieldConfig": {
...@@ -648,7 +615,6 @@ ...@@ -648,7 +615,6 @@
"mode": "palette-classic" "mode": "palette-classic"
}, },
"custom": { "custom": {
"axisBorderShow": false,
"axisCenteredZero": false, "axisCenteredZero": false,
"axisColorMode": "text", "axisColorMode": "text",
"axisLabel": "", "axisLabel": "",
...@@ -662,7 +628,6 @@ ...@@ -662,7 +628,6 @@
"tooltip": false, "tooltip": false,
"viz": false "viz": false
}, },
"insertNulls": false,
"lineInterpolation": "linear", "lineInterpolation": "linear",
"lineWidth": 1, "lineWidth": 1,
"pointSize": 5, "pointSize": 5,
...@@ -720,11 +685,11 @@ ...@@ -720,11 +685,11 @@
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "prometheus"
}, },
"disableTextWrap": false, "disableTextWrap": false,
"editorMode": "builder", "editorMode": "builder",
"expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket[$__rate_interval])))", "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
"fullMetaSearch": false, "fullMetaSearch": false,
"hide": false, "hide": false,
"includeNullMetadata": false, "includeNullMetadata": false,
...@@ -737,11 +702,11 @@ ...@@ -737,11 +702,11 @@
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "prometheus"
}, },
"disableTextWrap": false, "disableTextWrap": false,
"editorMode": "builder", "editorMode": "builder",
"expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket[$__rate_interval])))", "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
"fullMetaSearch": false, "fullMetaSearch": false,
"includeNullMetadata": false, "includeNullMetadata": false,
"instant": false, "instant": false,
...@@ -753,11 +718,11 @@ ...@@ -753,11 +718,11 @@
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "prometheus"
}, },
"disableTextWrap": false, "disableTextWrap": false,
"editorMode": "builder", "editorMode": "builder",
"expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket[$__rate_interval])))", "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
"fullMetaSearch": false, "fullMetaSearch": false,
"hide": false, "hide": false,
"includeNullMetadata": false, "includeNullMetadata": false,
...@@ -770,11 +735,11 @@ ...@@ -770,11 +735,11 @@
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "prometheus"
}, },
"disableTextWrap": false, "disableTextWrap": false,
"editorMode": "builder", "editorMode": "builder",
"expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket[$__rate_interval])))", "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
"fullMetaSearch": false, "fullMetaSearch": false,
"hide": false, "hide": false,
"includeNullMetadata": false, "includeNullMetadata": false,
...@@ -787,10 +752,10 @@ ...@@ -787,10 +752,10 @@
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "prometheus"
}, },
"editorMode": "code", "editorMode": "code",
"expr": "rate(vllm:time_to_first_token_seconds_sum[$__rate_interval])\n/\nrate(vllm:time_to_first_token_seconds_count[$__rate_interval])", "expr": "rate(vllm:time_to_first_token_seconds_sum{model_name=\"$model_name\"}[$__rate_interval])\n/\nrate(vllm:time_to_first_token_seconds_count{model_name=\"$model_name\"}[$__rate_interval])",
"hide": false, "hide": false,
"instant": false, "instant": false,
"legendFormat": "Average", "legendFormat": "Average",
...@@ -804,7 +769,7 @@ ...@@ -804,7 +769,7 @@
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "prometheus"
}, },
"description": "Percentage of used cache blocks by vLLM.", "description": "Percentage of used cache blocks by vLLM.",
"fieldConfig": { "fieldConfig": {
...@@ -813,7 +778,6 @@ ...@@ -813,7 +778,6 @@
"mode": "palette-classic" "mode": "palette-classic"
}, },
"custom": { "custom": {
"axisBorderShow": false,
"axisCenteredZero": false, "axisCenteredZero": false,
"axisColorMode": "text", "axisColorMode": "text",
"axisLabel": "", "axisLabel": "",
...@@ -827,7 +791,6 @@ ...@@ -827,7 +791,6 @@
"tooltip": false, "tooltip": false,
"viz": false "viz": false
}, },
"insertNulls": false,
"lineInterpolation": "linear", "lineInterpolation": "linear",
"lineWidth": 1, "lineWidth": 1,
"pointSize": 5, "pointSize": 5,
...@@ -885,10 +848,10 @@ ...@@ -885,10 +848,10 @@
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "prometheus"
}, },
"editorMode": "code", "editorMode": "code",
"expr": "vllm:gpu_cache_usage_perc", "expr": "vllm:gpu_cache_usage_perc{model_name=\"$model_name\"}",
"instant": false, "instant": false,
"legendFormat": "GPU Cache Usage", "legendFormat": "GPU Cache Usage",
"range": true, "range": true,
...@@ -897,10 +860,10 @@ ...@@ -897,10 +860,10 @@
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "prometheus"
}, },
"editorMode": "code", "editorMode": "code",
"expr": "vllm:cpu_cache_usage_perc", "expr": "vllm:cpu_cache_usage_perc{model_name=\"$model_name\"}",
"hide": false, "hide": false,
"instant": false, "instant": false,
"legendFormat": "CPU Cache Usage", "legendFormat": "CPU Cache Usage",
...@@ -913,10 +876,39 @@ ...@@ -913,10 +876,39 @@
} }
], ],
"refresh": "", "refresh": "",
"schemaVersion": 39, "schemaVersion": 37,
"style": "dark",
"tags": [], "tags": [],
"templating": { "templating": {
"list": [] "list": [
{
"current": {
"selected": false,
"text": "vllm",
"value": "vllm"
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"definition": "label_values(model_name)",
"hide": 0,
"includeAll": false,
"label": "model_name",
"multi": false,
"name": "model_name",
"options": [],
"query": {
"query": "label_values(model_name)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"type": "query"
}
]
}, },
"time": { "time": {
"from": "now-5m", "from": "now-5m",
......
examples/template_baichuan.jinja
View file @ 7c4f76e3
{{ (messages|selectattr('role', 'equalto', 'system')|list|last).content|trim if (messages|selectattr('role', 'equalto', 'system')|list) else '' }} {{ (messages|selectattr('role', 'equalto', 'system')|list|last).content|trim if (messages|selectattr('role', 'equalto', 'system')|list) else '' }}
{% for message in messages %} {%- for message in messages -%}
{% if message['role'] == 'user' %} {%- if message['role'] == 'user' -%}
<reserved_106> {{- '<reserved_106>' + message['content'] -}}
{{ message['content']|trim -}} {%- elif message['role'] == 'assistant' -%}
{% if not loop.last %} {{- '<reserved_107>' + message['content'] -}}
{%- endif -%}
{%- endfor -%}
{% endif %}
{% elif message['role'] == 'assistant' %} {%- if add_generation_prompt and messages[-1]['role'] != 'assistant' -%}
<reserved_107> {{- '<reserved_107>' -}}
{{ message['content']|trim -}}
{% if not loop.last %}
{% endif %}
{% endif %}
{% endfor %}
{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}
<reserved_107>
{% endif %} {% endif %}
\ No newline at end of file
examples/template_chatglm.jinja 0 → 100644
View file @ 7c4f76e3
{%- set counter = namespace(index=0) -%}
{%- for message in messages -%}
{%- if message['role'] == 'user' -%}
{{- '[Round ' + counter.index|string + ']\n问：' + message['content'] -}}
{%- set counter.index = counter.index + 1 -%}
{%- endif -%}
{%- if message['role'] == 'assistant' -%}
{{- '\n答：' + message['content'] -}}
{%- if (loop.last and add_generation_prompt) or not loop.last -%}
{{- '\n' -}}
{%- endif -%}
{%- endif -%}
{%- endfor -%}
{%- if add_generation_prompt and messages[-1]['role'] != 'assistant' -%}
{{- '\n答：' -}}
{%- endif -%}
\ No newline at end of file
examples/template_chatglm2.jinja 0 → 100644
View file @ 7c4f76e3
{%- set counter = namespace(index=1) -%}
{%- for message in messages -%}
{%- if message['role'] == 'user' -%}
{{- '[Round ' + counter.index|string + ']\n\n问：' + message['content'] -}}
{%- set counter.index = counter.index + 1 -%}
{%- endif -%}
{%- if message['role'] == 'assistant' -%}
{{- '\n\n答：' + message['content'] -}}
{%- if (loop.last and add_generation_prompt) or not loop.last -%}
{{- '\n\n' -}}
{%- endif -%}
{%- endif -%}
{%- endfor -%}
{%- if add_generation_prompt and messages[-1]['role'] != 'assistant' -%}
{{- '\n\n答：' -}}
{%- endif -%}
\ No newline at end of file
examples/template_falcon.jinja 0 → 100644
View file @ 7c4f76e3
{%- for message in messages -%}
{%- if message['role'] == 'user' -%}
{{- 'User: ' + message['content'] -}}
{%- elif message['role'] == 'assistant' -%}
{{- 'Assistant: ' + message['content'] -}}
{%- endif -%}
{%- if (loop.last and add_generation_prompt) or not loop.last -%}
{{- '\n' -}}
{%- endif -%}
{%- endfor -%}
{%- if add_generation_prompt and messages[-1]['role'] != 'assistant' -%}
{{- 'Assistant:' -}}
{% endif %}
\ No newline at end of file
examples/template_falcon_180b.jinja 0 → 100644
View file @ 7c4f76e3
{%- for message in messages -%}
{%- if message['role'] == 'system' -%}
{{- 'System: ' + message['content'] -}}
{%- elif message['role'] == 'user' -%}
{{- 'User: ' + message['content'] -}}
{%- elif message['role'] == 'assistant' -%}
{{- 'Falcon: ' + message['content'] -}}
{%- endif -%}
{%- if (loop.last and add_generation_prompt) or not loop.last -%}
{{- '\n' -}}
{%- endif -%}
{%- endfor -%}
{%- if add_generation_prompt and messages[-1]['role'] != 'assistant' -%}
{{- 'Falcon:' -}}
{% endif %}
\ No newline at end of file
format.sh
View file @ 7c4f76e3
...@@ -25,6 +25,7 @@ YAPF_VERSION=$(yapf --version | awk '{print $2}') ...@@ -25,6 +25,7 @@ YAPF_VERSION=$(yapf --version | awk '{print $2}')
RUFF_VERSION=$(ruff --version | awk '{print $2}') RUFF_VERSION=$(ruff --version | awk '{print $2}')
MYPY_VERSION=$(mypy --version | awk '{print $2}') MYPY_VERSION=$(mypy --version | awk '{print $2}')
CODESPELL_VERSION=$(codespell --version) CODESPELL_VERSION=$(codespell --version)
ISORT_VERSION=$(isort --vn)
# # params: tool name, tool version, required version # # params: tool name, tool version, required version
tool_version_check() { tool_version_check() {
...@@ -37,6 +38,7 @@ tool_version_check() { ...@@ -37,6 +38,7 @@ tool_version_check() {
tool_version_check "yapf" $YAPF_VERSION "$(grep yapf requirements-dev.txt | cut -d'=' -f3)" tool_version_check "yapf" $YAPF_VERSION "$(grep yapf requirements-dev.txt | cut -d'=' -f3)"
tool_version_check "ruff" $RUFF_VERSION "$(grep "ruff==" requirements-dev.txt | cut -d'=' -f3)" tool_version_check "ruff" $RUFF_VERSION "$(grep "ruff==" requirements-dev.txt | cut -d'=' -f3)"
tool_version_check "mypy" "$MYPY_VERSION" "$(grep mypy requirements-dev.txt | cut -d'=' -f3)" tool_version_check "mypy" "$MYPY_VERSION" "$(grep mypy requirements-dev.txt | cut -d'=' -f3)"
tool_version_check "isort" "$ISORT_VERSION" "$(grep isort requirements-dev.txt | cut -d'=' -f3)"
tool_version_check "codespell" "$CODESPELL_VERSION" "$(grep codespell requirements-dev.txt | cut -d'=' -f3)" tool_version_check "codespell" "$CODESPELL_VERSION" "$(grep codespell requirements-dev.txt | cut -d'=' -f3)"
YAPF_FLAGS=( YAPF_FLAGS=(
...@@ -95,13 +97,17 @@ echo 'vLLM yapf: Done' ...@@ -95,13 +97,17 @@ echo 'vLLM yapf: Done'
# echo 'vLLM mypy:' # echo 'vLLM mypy:'
# mypy # mypy
CODESPELL_EXCLUDES=(
'--skip' '*docs/source/_build/**'
)
# check spelling of specified files # check spelling of specified files
spell_check() { spell_check() {
codespell "$@" codespell "$@"
} }
spell_check_all(){ spell_check_all(){
codespell --toml pyproject.toml codespell --toml pyproject.toml "${CODESPELL_EXCLUDES[@]}"
} }
# Spelling check of files that differ from main branch. # Spelling check of files that differ from main branch.
...@@ -116,7 +122,7 @@ spell_check_changed() { ...@@ -116,7 +122,7 @@ spell_check_changed() {
if ! git diff --diff-filter=ACM --quiet --exit-code "$MERGEBASE" -- '*.py' '*.pyi' &>/dev/null; then if ! git diff --diff-filter=ACM --quiet --exit-code "$MERGEBASE" -- '*.py' '*.pyi' &>/dev/null; then
git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.py' '*.pyi' | xargs \ git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.py' '*.pyi' | xargs \
codespell codespell "${CODESPELL_EXCLUDES[@]}"
fi fi
} }
...@@ -174,6 +180,46 @@ else ...@@ -174,6 +180,46 @@ else
lint_changed lint_changed
fi fi
# check spelling of specified files
isort_check() {
isort "$@"
}
isort_check_all(){
isort .
}
# Spelling check of files that differ from main branch.
isort_check_changed() {
# The `if` guard ensures that the list of filenames is not empty, which
# could cause ruff to receive 0 positional arguments, making it hang
# waiting for STDIN.
#
# `diff-filter=ACM` and $MERGEBASE is to ensure we only lint files that
# exist on both branches.
MERGEBASE="$(git merge-base origin/main HEAD)"
if ! git diff --diff-filter=ACM --quiet --exit-code "$MERGEBASE" -- '*.py' '*.pyi' &>/dev/null; then
git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.py' '*.pyi' | xargs \
isort
fi
}
# Run Isort
# This flag runs spell check of individual files. --files *must* be the first command line
# arg to use this option.
if [[ "$1" == '--files' ]]; then
isort_check "${@:2}"
# If `--all` is passed, then any further arguments are ignored and the
# entire python directory is linted.
elif [[ "$1" == '--all' ]]; then
isort_check_all
else
# Check spelling only of the files that changed in last commit.
isort_check_changed
fi
echo 'vLLM isort: Done'
if ! git diff --quiet &>/dev/null; then if ! git diff --quiet &>/dev/null; then
echo 'Reformatted files. Please review and stage the changes.' echo 'Reformatted files. Please review and stage the changes.'
echo 'Changes not staged for commit:' echo 'Changes not staged for commit:'
......
pyproject.toml
View file @ 7c4f76e3
[build-system] [build-system]
# Should be mirrored in requirements-build.txt # Should be mirrored in requirements-build.txt
requires = [ requires = [
"cmake>=3.21",
"ninja", "ninja",
"packaging", "packaging",
"setuptools >= 49.4.0", "setuptools >= 49.4.0",
...@@ -9,6 +10,10 @@ requires = [ ...@@ -9,6 +10,10 @@ requires = [
] ]
build-backend = "setuptools.build_meta" build-backend = "setuptools.build_meta"
[tool.ruff]
# Allow lines to be as long as 80.
line-length = 80
[tool.ruff.lint] [tool.ruff.lint]
select = [ select = [
# pycodestyle # pycodestyle
...@@ -29,10 +34,6 @@ ignore = [ ...@@ -29,10 +34,6 @@ ignore = [
"F405", "F403", "F405", "F403",
# lambda expression assignment # lambda expression assignment
"E731", "E731",
# line too long, handled by black formatting
"E501",
# .strip() with multi-character strings
"B005",
# Loop control variable not used within loop body # Loop control variable not used within loop body
"B007", "B007",
] ]
...@@ -49,4 +50,8 @@ exclude = "vllm/model_executor/parallel_utils/|vllm/model_executor/models/" ...@@ -49,4 +50,8 @@ exclude = "vllm/model_executor/parallel_utils/|vllm/model_executor/models/"
[tool.codespell] [tool.codespell]
ignore-words-list = "dout, te, indicies" ignore-words-list = "dout, te, indicies"
skip = "./tests/prompts" skip = "./tests/prompts,./benchmarks/sonnet.txt"
[tool.isort]
use_parentheses = true
skip_gitignore = true
requirements-build.txt
View file @ 7c4f76e3
# Should be mirrored in pyproject.toml # Should be mirrored in pyproject.toml
cmake>=3.21
ninja ninja
packaging packaging
setuptools>=49.4.0 setuptools>=49.4.0
torch==2.1.2 torch==2.1.2
wheel wheel
\ No newline at end of file
requirements-dev.txt
View file @ 7c4f76e3
...@@ -4,6 +4,7 @@ toml==0.10.2 ...@@ -4,6 +4,7 @@ toml==0.10.2
tomli==2.0.1 tomli==2.0.1
ruff==0.1.5 ruff==0.1.5
codespell==2.2.6 codespell==2.2.6
isort==5.13.2
# type checking # type checking
mypy==0.991 mypy==0.991
...@@ -16,8 +17,18 @@ pytest ...@@ -16,8 +17,18 @@ pytest
pytest-forked pytest-forked
pytest-asyncio pytest-asyncio
pytest-rerunfailures pytest-rerunfailures
pytest-shard
httpx httpx
einops # required for MPT einops # required for MPT
openai openai
requests requests
ray ray
peft
awscli
ai2-olmo # required for OLMo
# Benchmarking
aiohttp
# Multimodal
pillow
requirements-neuron.txt
View file @ 7c4f76e3
...@@ -7,3 +7,6 @@ fastapi ...@@ -7,3 +7,6 @@ fastapi
uvicorn[standard] uvicorn[standard]
pydantic >= 2.0 # Required for OpenAI server. pydantic >= 2.0 # Required for OpenAI server.
prometheus_client >= 0.18.0 prometheus_client >= 0.18.0
requests
psutil
py-cpuinfo
\ No newline at end of file
requirements-rocm.txt
View file @ 7c4f76e3
cmake>=3.21
ninja # For faster builds. ninja # For faster builds.
typing-extensions>=4.8.0 typing-extensions>=4.8.0
starlette starlette
requests
py-cpuinfo
psutil psutil
ray >= 2.9 ray == 2.9.3
sentencepiece # Required for LLaMA tokenizer. sentencepiece # Required for LLaMA tokenizer.
numpy numpy
tokenizers>=0.15.0 tokenizers>=0.15.0
transformers >= 4.38.0 # Required for Gemma. transformers >= 4.39.1 # Required for StarCoder2 & Llava.
fastapi fastapi
uvicorn[standard] uvicorn[standard]
pydantic >= 2.0 # Required for OpenAI server. pydantic >= 2.0 # Required for OpenAI server.
prometheus_client >= 0.18.0 prometheus_client >= 0.18.0
outlines == 0.0.34
requirements.txt
View file @ 7c4f76e3
cmake>=3.21
ninja # For faster builds. ninja # For faster builds.
psutil psutil
ray >= 2.9 ray >= 2.9
sentencepiece # Required for LLaMA tokenizer. sentencepiece # Required for LLaMA tokenizer.
numpy numpy
torch == 2.1.2 torch == 2.1.2
transformers >= 4.38.0 # Required for Gemma. requests
psutil
py-cpuinfo
transformers >= 4.39.1 # Required for StarCoder2 & Llava.
xformers == 0.0.23.post1 # Required for CUDA 12.1. xformers == 0.0.23.post1 # Required for CUDA 12.1.
fastapi fastapi
uvicorn[standard] uvicorn[standard]
...@@ -12,5 +16,5 @@ pydantic >= 2.0 # Required for OpenAI server. ...@@ -12,5 +16,5 @@ pydantic >= 2.0 # Required for OpenAI server.
prometheus_client >= 0.18.0 prometheus_client >= 0.18.0
pynvml == 11.5.0 pynvml == 11.5.0
triton >= 2.1.0 triton >= 2.1.0
outlines >= 0.0.27 outlines == 0.0.34
cupy-cuda12x == 12.1.0 # Required for CUDA graphs. CUDA 11.8 users should install cupy-cuda11x instead. tiktoken == 0.6.0 # Required for DBRX tokenizer
setup.py
View file @ 7c4f76e3
import contextlib
import io import io
import logging
import os import os
import re import re
import subprocess import subprocess
import warnings import sys
from pathlib import Path from shutil import which
from typing import List, Set from typing import List
from packaging.version import parse, Version
import setuptools
import torch import torch
import torch.utils.cpp_extension as torch_cpp_ext from packaging.version import Version, parse
from torch.utils.cpp_extension import BuildExtension, CUDAExtension, CUDA_HOME, ROCM_HOME from setuptools import Extension, find_packages, setup
from setuptools.command.build_ext import build_ext
from torch.utils.cpp_extension import CUDA_HOME
from typing import Optional, Union from typing import Optional, Union
import subprocess import subprocess
from pathlib import Path from pathlib import Path
ROOT_DIR = os.path.dirname(__file__) ROOT_DIR = os.path.dirname(__file__)
logger = logging.getLogger(__name__)
# If you are developing the C++ backend of vLLM, consider building vLLM with # vLLM only supports Linux platform
# `python setup.py develop` since it will give you incremental builds. assert sys.platform.startswith(
# The downside is that this method is deprecated, see "linux"), "vLLM only supports Linux platform (including WSL)."
# https://github.com/pypa/setuptools/issues/917
MAIN_CUDA_VERSION = "12.1" MAIN_CUDA_VERSION = "12.1"
# Supported NVIDIA GPU architectures.
NVIDIA_SUPPORTED_ARCHS = {"7.0", "7.5", "8.0", "8.6", "8.9", "9.0"} def is_sccache_available() -> bool:
ROCM_SUPPORTED_ARCHS = {"gfx908", "gfx90a", "gfx906", "gfx926", "gfx928", "gfx936","gfx942", "gfx1100"} return which("sccache") is not None
# SUPPORTED_ARCHS = NVIDIA_SUPPORTED_ARCHS.union(ROCM_SUPPORTED_ARCHS)
def is_ccache_available() -> bool:
return which("ccache") is not None
def is_ninja_available() -> bool:
return which("ninja") is not None
def remove_prefix(text, prefix):
if text.startswith(prefix):
return text[len(prefix):]
return text
class CMakeExtension(Extension):
def __init__(self, name: str, cmake_lists_dir: str = '.', **kwa) -> None:
super().__init__(name, sources=[], **kwa)
self.cmake_lists_dir = os.path.abspath(cmake_lists_dir)
class cmake_build_ext(build_ext):
# A dict of extension directories that have been configured.
did_config = {}
#
# Determine number of compilation jobs and optionally nvcc compile threads.
#
def compute_num_jobs(self):
# `num_jobs` is either the value of the MAX_JOBS environment variable
# (if defined) or the number of CPUs available.
num_jobs = os.environ.get("MAX_JOBS", None)
if num_jobs is not None:
num_jobs = int(num_jobs)
logger.info(f"Using MAX_JOBS={num_jobs} as the number of jobs.")
else:
try:
# os.sched_getaffinity() isn't universally available, so fall
# back to os.cpu_count() if we get an error here.
num_jobs = len(os.sched_getaffinity(0))
except AttributeError:
num_jobs = os.cpu_count()
nvcc_threads = None
if _is_cuda() and get_nvcc_cuda_version() >= Version("11.2"):
# `nvcc_threads` is either the value of the NVCC_THREADS
# environment variable (if defined) or 1.
# when it is set, we reduce `num_jobs` to avoid
# overloading the system.
nvcc_threads = os.getenv("NVCC_THREADS", None)
if nvcc_threads is not None:
nvcc_threads = int(nvcc_threads)
logger.info(f"Using NVCC_THREADS={nvcc_threads} as the number"
" of nvcc threads.")
else:
nvcc_threads = 1
num_jobs = max(1, num_jobs // nvcc_threads)
return num_jobs, nvcc_threads
#
# Perform cmake configuration for a single extension.
#
def configure(self, ext: CMakeExtension) -> None:
# If we've already configured using the CMakeLists.txt for
# this extension, exit early.
if ext.cmake_lists_dir in cmake_build_ext.did_config:
return
cmake_build_ext.did_config[ext.cmake_lists_dir] = True
# Select the build type.
# Note: optimization level + debug info are set by the build type
default_cfg = "Debug" if self.debug else "RelWithDebInfo"
cfg = os.getenv("CMAKE_BUILD_TYPE", default_cfg)
# where .so files will be written, should be the same for all extensions
# that use the same CMakeLists.txt.
outdir = os.path.abspath(
os.path.dirname(self.get_ext_fullpath(ext.name)))
cmake_args = [
'-DCMAKE_BUILD_TYPE={}'.format(cfg),
'-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={}'.format(outdir),
'-DCMAKE_ARCHIVE_OUTPUT_DIRECTORY={}'.format(self.build_temp),
]
if _is_hip:
cmake_args += ['--gpu-max-threads-per-block=1024']
verbose = bool(int(os.getenv('VERBOSE', '0')))
if verbose:
cmake_args += ['-DCMAKE_VERBOSE_MAKEFILE=ON']
if is_sccache_available():
cmake_args += [
'-DCMAKE_CXX_COMPILER_LAUNCHER=sccache',
'-DCMAKE_CUDA_COMPILER_LAUNCHER=sccache',
]
elif is_ccache_available():
cmake_args += [
'-DCMAKE_CXX_COMPILER_LAUNCHER=ccache',
'-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache',
]
# Pass the python executable to cmake so it can find an exact
# match.
cmake_args += ['-DVLLM_PYTHON_EXECUTABLE={}'.format(sys.executable)]
if _install_punica():
cmake_args += ['-DVLLM_INSTALL_PUNICA_KERNELS=ON']
#
# Setup parallelism and build tool
#
num_jobs, nvcc_threads = self.compute_num_jobs()
if nvcc_threads:
cmake_args += ['-DNVCC_THREADS={}'.format(nvcc_threads)]
if is_ninja_available():
build_tool = ['-G', 'Ninja']
cmake_args += [
'-DCMAKE_JOB_POOL_COMPILE:STRING=compile',
'-DCMAKE_JOB_POOLS:STRING=compile={}'.format(num_jobs),
]
else:
# Default build tool to whatever cmake picks.
build_tool = []
subprocess.check_call(
['cmake', ext.cmake_lists_dir, *build_tool, *cmake_args],
cwd=self.build_temp)
def build_extensions(self) -> None:
# Ensure that CMake is present and working
try:
subprocess.check_output(['cmake', '--version'])
except OSError as e:
raise RuntimeError('Cannot find CMake executable') from e
# Create build directory if it does not exist.
if not os.path.exists(self.build_temp):
os.makedirs(self.build_temp)
# Build all the extensions
for ext in self.extensions:
self.configure(ext)
ext_target_name = remove_prefix(ext.name, "vllm.")
num_jobs, _ = self.compute_num_jobs()
build_args = [
'--build', '.', '--target', ext_target_name, '-j',
str(num_jobs)
]
subprocess.check_call(['cmake', *build_args], cwd=self.build_temp)
def _is_cuda() -> bool:
return torch.version.cuda is not None and not _is_neuron()
def _is_hip() -> bool: def _is_hip() -> bool:
...@@ -40,36 +203,13 @@ def _is_neuron() -> bool: ...@@ -40,36 +203,13 @@ def _is_neuron() -> bool:
torch_neuronx_installed = True torch_neuronx_installed = True
try: try:
subprocess.run(["neuron-ls"], capture_output=True, check=True) subprocess.run(["neuron-ls"], capture_output=True, check=True)
except (FileNotFoundError, PermissionError): except (FileNotFoundError, PermissionError, subprocess.CalledProcessError):
torch_neuronx_installed = False torch_neuronx_installed = False
return torch_neuronx_installed return torch_neuronx_installed
def _is_cuda() -> bool: def _install_punica() -> bool:
return (torch.version.cuda is not None) and not _is_neuron() return bool(int(os.getenv("VLLM_INSTALL_PUNICA_KERNELS", "0")))
# Compiler flags.
CXX_FLAGS = ["-g", "-O2", "-std=c++17"]
# TODO(woosuk): Should we use -O3?
NVCC_FLAGS = ["-O2", "-std=c++17","--gpu-max-threads-per-block=1024"]
if _is_hip():
if ROCM_HOME is None:
raise RuntimeError(
"Cannot find ROCM_HOME. ROCm must be available to build the package."
)
NVCC_FLAGS += ["-DUSE_ROCM"]
NVCC_FLAGS += ["-U__HIP_NO_HALF_CONVERSIONS__"]
NVCC_FLAGS += ["-U__HIP_NO_HALF_OPERATORS__"]
if _is_cuda() and CUDA_HOME is None:
raise RuntimeError(
"Cannot find CUDA_HOME. CUDA must be available to build the package.")
ABI = 1 if torch._C._GLIBCXX_USE_CXX11_ABI else 0
CXX_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
NVCC_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
def get_hipcc_rocm_version(): def get_hipcc_rocm_version():
...@@ -94,11 +234,6 @@ def get_hipcc_rocm_version(): ...@@ -94,11 +234,6 @@ def get_hipcc_rocm_version():
return None return None
def glob(pattern: str):
root = Path(__name__).parent
return [str(p) for p in root.glob(pattern)]
def get_neuronxcc_version(): def get_neuronxcc_version():
import sysconfig import sysconfig
site_dir = sysconfig.get_paths()["purelib"] site_dir = sysconfig.get_paths()["purelib"]
...@@ -118,12 +253,12 @@ def get_neuronxcc_version(): ...@@ -118,12 +253,12 @@ def get_neuronxcc_version():
raise RuntimeError("Could not find HIP version in the output") raise RuntimeError("Could not find HIP version in the output")
def get_nvcc_cuda_version(cuda_dir: str) -> Version: def get_nvcc_cuda_version() -> Version:
"""Get the CUDA version from nvcc. """Get the CUDA version from nvcc.
Adapted from https://github.com/NVIDIA/apex/blob/8b7a1ff183741dd8f9b87e7bafd04cfde99cea28/setup.py Adapted from https://github.com/NVIDIA/apex/blob/8b7a1ff183741dd8f9b87e7bafd04cfde99cea28/setup.py
""" """
nvcc_output = subprocess.check_output([cuda_dir + "/bin/nvcc", "-V"], nvcc_output = subprocess.check_output([CUDA_HOME + "/bin/nvcc", "-V"],
universal_newlines=True) universal_newlines=True)
output = nvcc_output.split() output = nvcc_output.split()
release_idx = output.index("release") + 1 release_idx = output.index("release") + 1
...@@ -131,249 +266,6 @@ def get_nvcc_cuda_version(cuda_dir: str) -> Version: ...@@ -131,249 +266,6 @@ def get_nvcc_cuda_version(cuda_dir: str) -> Version:
return nvcc_cuda_version return nvcc_cuda_version
def get_pytorch_rocm_arch() -> Set[str]:
"""Get the cross section of Pytorch,and vllm supported gfx arches
ROCM can get the supported gfx architectures in one of two ways
Either through the PYTORCH_ROCM_ARCH env var, or output from
rocm_agent_enumerator.
In either case we can generate a list of supported arch's and
cross reference with VLLM's own ROCM_SUPPORTED_ARCHs.
"""
env_arch_list = os.environ.get("PYTORCH_ROCM_ARCH", None)
# If we don't have PYTORCH_ROCM_ARCH specified pull the list from rocm_agent_enumerator
if env_arch_list is None:
command = "rocm_agent_enumerator"
env_arch_list = subprocess.check_output([command]).decode('utf-8')\
.strip().replace("\n", ";")
arch_source_str = "rocm_agent_enumerator"
else:
arch_source_str = "PYTORCH_ROCM_ARCH env variable"
# List are separated by ; or space.
pytorch_rocm_arch = set(env_arch_list.replace(" ", ";").split(";"))
# Filter out the invalid architectures and print a warning.
arch_list = pytorch_rocm_arch.intersection(ROCM_SUPPORTED_ARCHS)
# If none of the specified architectures are valid, raise an error.
if not arch_list:
raise RuntimeError(
f"None of the ROCM architectures in {arch_source_str} "
f"({env_arch_list}) is supported. "
f"Supported ROCM architectures are: {ROCM_SUPPORTED_ARCHS}.")
invalid_arch_list = pytorch_rocm_arch - ROCM_SUPPORTED_ARCHS
if invalid_arch_list:
warnings.warn(
f"Unsupported ROCM architectures ({invalid_arch_list}) are "
f"excluded from the {arch_source_str} output "
f"({env_arch_list}). Supported ROCM architectures are: "
f"{ROCM_SUPPORTED_ARCHS}.",
stacklevel=2)
return arch_list
def get_torch_arch_list() -> Set[str]:
# TORCH_CUDA_ARCH_LIST can have one or more architectures,
# e.g. "8.0" or "7.5,8.0,8.6+PTX". Here, the "8.6+PTX" option asks the
# compiler to additionally include PTX code that can be runtime-compiled
# and executed on the 8.6 or newer architectures. While the PTX code will
# not give the best performance on the newer architectures, it provides
# forward compatibility.
env_arch_list = os.environ.get("TORCH_CUDA_ARCH_LIST", None)
if env_arch_list is None:
return set()
# List are separated by ; or space.
torch_arch_list = set(env_arch_list.replace(" ", ";").split(";"))
if not torch_arch_list:
return set()
# Filter out the invalid architectures and print a warning.
valid_archs = NVIDIA_SUPPORTED_ARCHS.union(
{s + "+PTX"
for s in NVIDIA_SUPPORTED_ARCHS})
arch_list = torch_arch_list.intersection(valid_archs)
# If none of the specified architectures are valid, raise an error.
if not arch_list:
raise RuntimeError(
"None of the CUDA architectures in `TORCH_CUDA_ARCH_LIST` env "
f"variable ({env_arch_list}) is supported. "
f"Supported CUDA architectures are: {valid_archs}.")
invalid_arch_list = torch_arch_list - valid_archs
if invalid_arch_list:
warnings.warn(
f"Unsupported CUDA architectures ({invalid_arch_list}) are "
"excluded from the `TORCH_CUDA_ARCH_LIST` env variable "
f"({env_arch_list}). Supported CUDA architectures are: "
f"{valid_archs}.",
stacklevel=2)
return arch_list
if _is_hip():
rocm_arches = get_pytorch_rocm_arch()
NVCC_FLAGS += ["--offload-arch=" + arch for arch in rocm_arches]
else:
# First, check the TORCH_CUDA_ARCH_LIST environment variable.
compute_capabilities = get_torch_arch_list()
if _is_cuda() and not compute_capabilities:
# If TORCH_CUDA_ARCH_LIST is not defined or empty, target all available
# GPUs on the current machine.
device_count = torch.cuda.device_count()
for i in range(device_count):
major, minor = torch.cuda.get_device_capability(i)
if major < 7:
raise RuntimeError(
"GPUs with compute capability below 7.0 are not supported.")
compute_capabilities.add(f"{major}.{minor}")
ext_modules = []
if _is_cuda():
nvcc_cuda_version = get_nvcc_cuda_version(CUDA_HOME)
if not compute_capabilities:
# If no GPU is specified nor available, add all supported architectures
# based on the NVCC CUDA version.
compute_capabilities = NVIDIA_SUPPORTED_ARCHS.copy()
if nvcc_cuda_version < Version("11.1"):
compute_capabilities.remove("8.6")
if nvcc_cuda_version < Version("11.8"):
compute_capabilities.remove("8.9")
compute_capabilities.remove("9.0")
# Validate the NVCC CUDA version.
if nvcc_cuda_version < Version("11.0"):
raise RuntimeError(
"CUDA 11.0 or higher is required to build the package.")
if (nvcc_cuda_version < Version("11.1")
and any(cc.startswith("8.6") for cc in compute_capabilities)):
raise RuntimeError(
"CUDA 11.1 or higher is required for compute capability 8.6.")
if nvcc_cuda_version < Version("11.8"):
if any(cc.startswith("8.9") for cc in compute_capabilities):
# CUDA 11.8 is required to generate the code targeting compute capability 8.9.
# However, GPUs with compute capability 8.9 can also run the code generated by
# the previous versions of CUDA 11 and targeting compute capability 8.0.
# Therefore, if CUDA 11.8 is not available, we target compute capability 8.0
# instead of 8.9.
warnings.warn(
"CUDA 11.8 or higher is required for compute capability 8.9. "
"Targeting compute capability 8.0 instead.",
stacklevel=2)
compute_capabilities = set(cc for cc in compute_capabilities
if not cc.startswith("8.9"))
compute_capabilities.add("8.0+PTX")
if any(cc.startswith("9.0") for cc in compute_capabilities):
raise RuntimeError(
"CUDA 11.8 or higher is required for compute capability 9.0.")
NVCC_FLAGS_PUNICA = NVCC_FLAGS.copy()
# Add target compute capabilities to NVCC flags.
for capability in compute_capabilities:
num = capability[0] + capability[2]
NVCC_FLAGS += ["-gencode", f"arch=compute_{num},code=sm_{num}"]
if capability.endswith("+PTX"):
NVCC_FLAGS += [
"-gencode", f"arch=compute_{num},code=compute_{num}"
]
if int(capability[0]) >= 8:
NVCC_FLAGS_PUNICA += [
"-gencode", f"arch=compute_{num},code=sm_{num}"
]
if capability.endswith("+PTX"):
NVCC_FLAGS_PUNICA += [
"-gencode", f"arch=compute_{num},code=compute_{num}"
]
# Use NVCC threads to parallelize the build.
if nvcc_cuda_version >= Version("11.2"):
nvcc_threads = int(os.getenv("NVCC_THREADS", 8))
num_threads = min(os.cpu_count(), nvcc_threads)
NVCC_FLAGS += ["--threads", str(num_threads)]
if nvcc_cuda_version >= Version("11.8"):
NVCC_FLAGS += ["-DENABLE_FP8_E5M2"]
# changes for punica kernels
NVCC_FLAGS += torch_cpp_ext.COMMON_NVCC_FLAGS
REMOVE_NVCC_FLAGS = [
'-D__CUDA_NO_HALF_OPERATORS__',
'-D__CUDA_NO_HALF_CONVERSIONS__',
'-D__CUDA_NO_BFLOAT16_CONVERSIONS__',
'-D__CUDA_NO_HALF2_OPERATORS__',
]
for flag in REMOVE_NVCC_FLAGS:
with contextlib.suppress(ValueError):
torch_cpp_ext.COMMON_NVCC_FLAGS.remove(flag)
install_punica = bool(int(os.getenv("VLLM_INSTALL_PUNICA_KERNELS", "0")))
device_count = torch.cuda.device_count()
for i in range(device_count):
major, minor = torch.cuda.get_device_capability(i)
if major < 8:
install_punica = False
break
if install_punica:
ext_modules.append(
CUDAExtension(
name="vllm._punica_C",
sources=["csrc/punica/punica_ops.cc"] +
glob("csrc/punica/bgmv/*.cu"),
extra_compile_args={
"cxx": CXX_FLAGS,
"nvcc": NVCC_FLAGS_PUNICA,
},
))
elif _is_neuron():
neuronxcc_version = get_neuronxcc_version()
vllm_extension_sources = [
"csrc/cache_kernels.cu",
"csrc/attention/attention_kernels.cu",
"csrc/pos_encoding_kernels.cu",
"csrc/activation_kernels.cu",
"csrc/layernorm_kernels.cu",
"csrc/quantization/squeezellm/quant_cuda_kernel.cu",
"csrc/quantization/gptq/q_gemm.cu",
"csrc/cuda_utils_kernels.cu",
"csrc/moe_align_block_size_kernels.cu",
"csrc/pybind.cpp",
]
if _is_cuda():
vllm_extension_sources.append("csrc/quantization/awq/gemm_kernels.cu")
vllm_extension_sources.append(
"csrc/quantization/marlin/marlin_cuda_kernel.cu")
vllm_extension_sources.append("csrc/custom_all_reduce.cu")
# Add MoE kernels.
ext_modules.append(
CUDAExtension(
name="vllm._moe_C",
sources=glob("csrc/moe/*.cu") + glob("csrc/moe/*.cpp"),
extra_compile_args={
"cxx": CXX_FLAGS,
"nvcc": NVCC_FLAGS,
},
))
if not _is_neuron():
vllm_extension = CUDAExtension(
name="vllm._C",
sources=vllm_extension_sources,
extra_compile_args={
"cxx": CXX_FLAGS,
"nvcc": NVCC_FLAGS,
},
libraries=["cuda"] if _is_cuda() else [],
)
ext_modules.append(vllm_extension)
def get_path(*filepath) -> str: def get_path(*filepath) -> str:
return os.path.join(ROOT_DIR, *filepath) return os.path.join(ROOT_DIR, *filepath)
...@@ -432,8 +324,8 @@ def get_version_add(sha: Optional[str] = None) -> str: ...@@ -432,8 +324,8 @@ def get_version_add(sha: Optional[str] = None) -> str:
version += ".torch" + torch.__version__[:3] version += ".torch" + torch.__version__[:3]
with open(add_version_path, encoding="utf-8",mode="w") as file: with open(add_version_path, encoding="utf-8",mode="w") as file:
file.write("__version__='0.3.3'\n") file.write("__version__='0.4.0'\n")
file.write("__dcu_version__='0.3.3+{}'\n".format(version)) file.write("__dcu_version__='0.4.0+{}'\n".format(version))
file.close() file.close()
...@@ -448,7 +340,12 @@ def get_version(): ...@@ -448,7 +340,12 @@ def get_version():
def get_vllm_version() -> str: def get_vllm_version() -> str:
version = find_version(get_path("vllm", "__init__.py")) version = find_version(get_path("vllm", "__init__.py"))
if _is_hip(): if _is_cuda():
cuda_version = str(get_nvcc_cuda_version())
if cuda_version != MAIN_CUDA_VERSION:
cuda_version_str = cuda_version.replace(".", "")[:3]
version += f"+cu{cuda_version_str}"
elif _is_hip():
# Get the HIP version # Get the HIP version
hipcc_version = get_hipcc_rocm_version() hipcc_version = get_hipcc_rocm_version()
if hipcc_version != MAIN_CUDA_VERSION: if hipcc_version != MAIN_CUDA_VERSION:
...@@ -457,15 +354,12 @@ def get_vllm_version() -> str: ...@@ -457,15 +354,12 @@ def get_vllm_version() -> str:
version = get_version() version = get_version()
elif _is_neuron(): elif _is_neuron():
# Get the Neuron version # Get the Neuron version
neuron_version = str(neuronxcc_version) neuron_version = str(get_neuronxcc_version())
if neuron_version != MAIN_CUDA_VERSION: if neuron_version != MAIN_CUDA_VERSION:
neuron_version_str = neuron_version.replace(".", "")[:3] neuron_version_str = neuron_version.replace(".", "")[:3]
version += f"+neuron{neuron_version_str}" version += f"+neuron{neuron_version_str}"
else: else:
cuda_version = str(nvcc_cuda_version) raise RuntimeError("Unknown runtime environment")
if cuda_version != MAIN_CUDA_VERSION:
cuda_version_str = cuda_version.replace(".", "")[:3]
version += f"+cu{cuda_version_str}"
return version return version
...@@ -481,26 +375,40 @@ def read_readme() -> str: ...@@ -481,26 +375,40 @@ def read_readme() -> str:
def get_requirements() -> List[str]: def get_requirements() -> List[str]:
"""Get Python package dependencies from requirements.txt.""" """Get Python package dependencies from requirements.txt."""
if _is_hip(): if _is_cuda():
with open(get_path("requirements.txt")) as f:
requirements = f.read().strip().split("\n")
elif _is_hip():
with open(get_path("requirements-rocm.txt")) as f: with open(get_path("requirements-rocm.txt")) as f:
requirements = f.read().strip().split("\n") requirements = f.read().strip().split("\n")
elif _is_neuron(): elif _is_neuron():
with open(get_path("requirements-neuron.txt")) as f: with open(get_path("requirements-neuron.txt")) as f:
requirements = f.read().strip().split("\n") requirements = f.read().strip().split("\n")
else: else:
with open(get_path("requirements.txt")) as f: raise ValueError(
requirements = f.read().strip().split("\n") "Unsupported platform, please use CUDA, ROCM or Neuron.")
return requirements return requirements
ext_modules = []
if _is_cuda():
ext_modules.append(CMakeExtension(name="vllm._moe_C"))
if _install_punica():
ext_modules.append(CMakeExtension(name="vllm._punica_C"))
if not _is_neuron():
ext_modules.append(CMakeExtension(name="vllm._C"))
package_data = { package_data = {
"vllm": ["py.typed", "model_executor/layers/fused_moe/configs/*.json"] "vllm": ["py.typed", "model_executor/layers/fused_moe/configs/*.json"]
} }
if os.environ.get("VLLM_USE_PRECOMPILED"): if os.environ.get("VLLM_USE_PRECOMPILED"):
ext_modules = []
package_data["vllm"].append("*.so") package_data["vllm"].append("*.so")
setuptools.setup( setup(
name="vllm", name="vllm",
version=get_vllm_version(), version=get_vllm_version(),
author="vLLM Team", author="vLLM Team",
...@@ -522,11 +430,11 @@ setuptools.setup( ...@@ -522,11 +430,11 @@ setuptools.setup(
"License :: OSI Approved :: Apache Software License", "License :: OSI Approved :: Apache Software License",
"Topic :: Scientific/Engineering :: Artificial Intelligence", "Topic :: Scientific/Engineering :: Artificial Intelligence",
], ],
packages=setuptools.find_packages(exclude=("benchmarks", "csrc", "docs", packages=find_packages(exclude=("benchmarks", "csrc", "docs", "examples",
"examples", "tests")), "tests")),
python_requires=">=3.8", python_requires=">=3.8",
install_requires=get_requirements(), install_requires=get_requirements(),
ext_modules=ext_modules, ext_modules=ext_modules,
cmdclass={"build_ext": BuildExtension} if not _is_neuron() else {}, cmdclass={"build_ext": cmake_build_ext} if not _is_neuron() else {},
package_data=package_data, package_data=package_data,
) )
tests/async_engine/test_api_server.py
View file @ 7c4f76e3
...@@ -25,23 +25,21 @@ def _query_server_long(prompt: str) -> dict: ...@@ -25,23 +25,21 @@ def _query_server_long(prompt: str) -> dict:
@pytest.fixture @pytest.fixture
def api_server(): def api_server(tokenizer_pool_size: int):
script_path = Path(__file__).parent.joinpath( script_path = Path(__file__).parent.joinpath(
"api_server_async_engine.py").absolute() "api_server_async_engine.py").absolute()
uvicorn_process = subprocess.Popen([ uvicorn_process = subprocess.Popen([
sys.executable, sys.executable, "-u",
"-u", str(script_path), "--model", "facebook/opt-125m", "--host",
str(script_path), "127.0.0.1", "--tokenizer-pool-size",
"--model", str(tokenizer_pool_size)
"facebook/opt-125m",
"--host",
"127.0.0.1",
]) ])
yield yield
uvicorn_process.terminate() uvicorn_process.terminate()
def test_api_server(api_server): @pytest.mark.parametrize("tokenizer_pool_size", [0, 2])
def test_api_server(api_server, tokenizer_pool_size: int):
""" """
Run the API server and test it. Run the API server and test it.
......
tests/async_engine/test_async_llm_engine.py
View file @ 7c4f76e3
...@@ -25,12 +25,8 @@ class MockEngine: ...@@ -25,12 +25,8 @@ class MockEngine:
return [RequestOutput( return [RequestOutput(
request_id=self.request_id)] if self.request_id else [] request_id=self.request_id)] if self.request_id else []
async def encode_request_async( async def encode_request_async(self, *args, **kwargs):
self, pass
*args,
**kwargs,
):
return [1]
def generate(self, request_id): def generate(self, request_id):
self.request_id = request_id self.request_id = request_id
...@@ -43,13 +39,16 @@ class MockEngine: ...@@ -43,13 +39,16 @@ class MockEngine:
self.add_request_calls += 1 self.add_request_calls += 1
async def add_request_async(self, **kwargs): async def add_request_async(self, **kwargs):
del kwargs # Unused
self.add_request_calls += 1 self.add_request_calls += 1
return
def abort_request(self, request_id): def abort_request(self, request_id):
del request_id # Unused del request_id # Unused
self.abort_request_calls += 1 self.abort_request_calls += 1
def has_unfinished_requests(self):
return self.request_id is not None
class MockAsyncLLMEngine(AsyncLLMEngine): class MockAsyncLLMEngine(AsyncLLMEngine):
...@@ -72,20 +71,24 @@ async def test_new_requests_event(): ...@@ -72,20 +71,24 @@ async def test_new_requests_event():
await engine.add_request("2", "", None) await engine.add_request("2", "", None)
engine.engine.generate("2") engine.engine.generate("2")
await asyncio.sleep(0) await asyncio.sleep(0)
assert engine.engine.add_request_calls == 2
assert engine.engine.step_calls == 2
await asyncio.sleep(0) await asyncio.sleep(0)
assert engine.engine.step_calls == 3 assert engine.engine.add_request_calls == 2
assert engine.engine.step_calls >= 2
await asyncio.sleep(0.001)
assert engine.engine.step_calls >= 3
engine.engine.stop_generating() engine.engine.stop_generating()
await asyncio.sleep(0) await asyncio.sleep(0.001)
assert engine.engine.step_calls == 4 old_step_calls = engine.engine.step_calls
await asyncio.sleep(0) await asyncio.sleep(0.001)
assert engine.engine.step_calls == 4 assert engine.engine.step_calls == old_step_calls
await engine.add_request("3", "", None) await engine.add_request("3", "", None)
await asyncio.sleep(0.01) await asyncio.sleep(0.01)
assert engine.engine.add_request_calls == 3 assert engine.engine.add_request_calls == 3
assert engine.engine.step_calls == 5 assert engine.engine.step_calls == old_step_calls + 1
await asyncio.sleep(0.01) await asyncio.sleep(0.01)
assert engine.engine.add_request_calls == 3 assert engine.engine.add_request_calls == 3
assert engine.engine.step_calls == 5 assert engine.engine.step_calls == old_step_calls + 1
engine = MockAsyncLLMEngine(worker_use_ray=True, engine_use_ray=True)
assert engine.get_tokenizer() is not None
tests/async_engine/test_chat_template.py
View file @ 7c4f76e3
from dataclasses import dataclass
import os import os
import pathlib import pathlib
from dataclasses import dataclass
import pytest import pytest
from vllm.transformers_utils.tokenizer import get_tokenizer
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
from vllm.entrypoints.openai.protocol import ChatCompletionRequest from vllm.entrypoints.openai.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
from vllm.transformers_utils.tokenizer import get_tokenizer
chatml_jinja_path = pathlib.Path(os.path.dirname(os.path.abspath( chatml_jinja_path = pathlib.Path(os.path.dirname(os.path.abspath(
__file__))).parent.parent / "examples/template_chatml.jinja" __file__))).parent.parent / "examples/template_chatml.jinja"
...@@ -73,7 +73,7 @@ def test_load_chat_template(): ...@@ -73,7 +73,7 @@ def test_load_chat_template():
assert template_content is not None assert template_content is not None
# Hard coded value for template_chatml.jinja # Hard coded value for template_chatml.jinja
assert template_content == """{% for message in messages %}{{'<|im_start|>' + message['role'] + '\\n' + message['content']}}{% if (loop.last and add_generation_prompt) or not loop.last %}{{ '<|im_end|>' + '\\n'}}{% endif %}{% endfor %} assert template_content == """{% for message in messages %}{{'<|im_start|>' + message['role'] + '\\n' + message['content']}}{% if (loop.last and add_generation_prompt) or not loop.last %}{{ '<|im_end|>' + '\\n'}}{% endif %}{% endfor %}
{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{ '<|im_start|>assistant\\n' }}{% endif %}""" {% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{ '<|im_start|>assistant\\n' }}{% endif %}""" # noqa: E501
def test_no_load_chat_template(): def test_no_load_chat_template():
...@@ -117,4 +117,6 @@ async def test_get_gen_prompt(model, template, add_generation_prompt, ...@@ -117,4 +117,6 @@ async def test_get_gen_prompt(model, template, add_generation_prompt,
add_generation_prompt=mock_request.add_generation_prompt) add_generation_prompt=mock_request.add_generation_prompt)
# Test assertion # Test assertion
assert result == expected_output, f"The generated prompt does not match the expected output for model {model} and template {template}" assert result == expected_output, (
f"The generated prompt does not match the expected output for "
f"model {model} and template {template}")
tests/async_engine/test_request_tracker.py
View file @ 7c4f76e3
...@@ -4,25 +4,14 @@ from vllm.engine.async_llm_engine import RequestTracker ...@@ -4,25 +4,14 @@ from vllm.engine.async_llm_engine import RequestTracker
from vllm.outputs import RequestOutput from vllm.outputs import RequestOutput
class DummyEvent: @pytest.mark.asyncio
async def test_request_tracker():
def __init__(self):
self.flag = False
def set(self):
self.flag = True
def clear(self):
self.flag = False
def test_request_tracker():
tracker = RequestTracker() tracker = RequestTracker()
tracker.new_requests_event = DummyEvent()
stream_1 = tracker.add_request("1") stream_1 = tracker.add_request("1")
assert tracker.new_requests_event.flag assert tracker.new_requests_event.is_set()
await tracker.wait_for_new_requests()
new, finished = tracker.get_new_and_finished_requests() new, finished = tracker.get_new_and_finished_requests()
assert not tracker.new_requests_event.flag assert not tracker.new_requests_event.is_set()
assert len(new) == 1 assert len(new) == 1
assert new[0]["request_id"] == "1" assert new[0]["request_id"] == "1"
assert not finished assert not finished
...@@ -30,9 +19,10 @@ def test_request_tracker(): ...@@ -30,9 +19,10 @@ def test_request_tracker():
stream_2 = tracker.add_request("2") stream_2 = tracker.add_request("2")
stream_3 = tracker.add_request("3") stream_3 = tracker.add_request("3")
assert tracker.new_requests_event.flag assert tracker.new_requests_event.is_set()
await tracker.wait_for_new_requests()
new, finished = tracker.get_new_and_finished_requests() new, finished = tracker.get_new_and_finished_requests()
assert not tracker.new_requests_event.flag assert not tracker.new_requests_event.is_set()
assert len(new) == 2 assert len(new) == 2
assert new[0]["request_id"] == "2" assert new[0]["request_id"] == "2"
assert new[1]["request_id"] == "3" assert new[1]["request_id"] == "3"
...@@ -43,7 +33,7 @@ def test_request_tracker(): ...@@ -43,7 +33,7 @@ def test_request_tracker():
# request_ids must be unique # request_ids must be unique
with pytest.raises(KeyError): with pytest.raises(KeyError):
tracker.add_request("1") tracker.add_request("1")
assert not tracker.new_requests_event.flag assert not tracker.new_requests_event.is_set()
tracker.abort_request("1") tracker.abort_request("1")
new, finished = tracker.get_new_and_finished_requests() new, finished = tracker.get_new_and_finished_requests()
...@@ -54,7 +44,8 @@ def test_request_tracker(): ...@@ -54,7 +44,8 @@ def test_request_tracker():
stream_4 = tracker.add_request("4") stream_4 = tracker.add_request("4")
tracker.abort_request("4") tracker.abort_request("4")
assert tracker.new_requests_event.flag assert tracker.new_requests_event.is_set()
await tracker.wait_for_new_requests()
new, finished = tracker.get_new_and_finished_requests() new, finished = tracker.get_new_and_finished_requests()
assert len(finished) == 1 assert len(finished) == 1
assert "4" in finished assert "4" in finished
...@@ -62,11 +53,12 @@ def test_request_tracker(): ...@@ -62,11 +53,12 @@ def test_request_tracker():
assert stream_4.finished assert stream_4.finished
stream_5 = tracker.add_request("5") stream_5 = tracker.add_request("5")
assert tracker.new_requests_event.flag assert tracker.new_requests_event.is_set()
tracker.process_request_output( tracker.process_request_output(
RequestOutput("2", "output", [], [], [], bool(finished))) RequestOutput("2", "output", [], [], [], finished=True))
await tracker.wait_for_new_requests()
new, finished = tracker.get_new_and_finished_requests() new, finished = tracker.get_new_and_finished_requests()
assert not tracker.new_requests_event.flag assert not tracker.new_requests_event.is_set()
assert len(finished) == 1 assert len(finished) == 1
assert "2" in finished assert "2" in finished
assert len(new) == 1 assert len(new) == 1
......
tests/basic_correctness/test_basic_correctness.py
View file @ 7c4f76e3
"""Compare the short outputs of HF and vLLM when using greedy sampling. """Compare the short outputs of HF and vLLM when using greedy sampling.
Run `pytest tests/basic_correctness/test_basic_correctness.py --forked`. Run `pytest tests/basic_correctness/test_basic_correctness.py`.
""" """
import pytest import pytest
...@@ -13,6 +13,7 @@ MODELS = [ ...@@ -13,6 +13,7 @@ MODELS = [
@pytest.mark.parametrize("model", MODELS) @pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["half"]) @pytest.mark.parametrize("dtype", ["half"])
@pytest.mark.parametrize("max_tokens", [5]) @pytest.mark.parametrize("max_tokens", [5])
@pytest.mark.parametrize("enforce_eager", [False, True])
def test_models( def test_models(
hf_runner, hf_runner,
vllm_runner, vllm_runner,
...@@ -20,12 +21,13 @@ def test_models( ...@@ -20,12 +21,13 @@ def test_models(
model: str, model: str,
dtype: str, dtype: str,
max_tokens: int, max_tokens: int,
enforce_eager: bool,
) -> None: ) -> None:
hf_model = hf_runner(model, dtype=dtype) hf_model = hf_runner(model, dtype=dtype)
hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens) hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
del hf_model del hf_model
vllm_model = vllm_runner(model, dtype=dtype) vllm_model = vllm_runner(model, dtype=dtype, enforce_eager=enforce_eager)
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens) vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
del vllm_model del vllm_model
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment