"vscode:/vscode.git/clone" did not exist on "8ed5421aaa7da24051acdae53c860e6ce6598403"
Unverified Commit a302a8fd authored by daiyu1111's avatar daiyu1111 Committed by GitHub
Browse files

[Bugfix] Fix LLM priority normalization for single-string prompts (#40011)


Signed-off-by: default avatardaiyu1111 <2356690121@qq.com>
parent 4e8c3f1c
...@@ -91,6 +91,12 @@ def test_multiple_priority(llm: LLM): ...@@ -91,6 +91,12 @@ def test_multiple_priority(llm: LLM):
outputs = llm.generate(PROMPTS, sampling_params=None, priority=[]) outputs = llm.generate(PROMPTS, sampling_params=None, priority=[])
def test_single_prompt_priority(llm: LLM):
# Single string prompts should be normalized to one request.
outputs = llm.generate(PROMPTS[0], sampling_params=None, priority=[0])
assert len(outputs) == 1
def test_max_model_len(): def test_max_model_len():
max_model_len = 20 max_model_len = 20
llm = LLM( llm = LLM(
......
...@@ -1575,7 +1575,7 @@ class LLM: ...@@ -1575,7 +1575,7 @@ class LLM:
seq_prompts = prompt_to_seq(prompts) seq_prompts = prompt_to_seq(prompts)
seq_params = self._params_to_seq(params, len(seq_prompts)) seq_params = self._params_to_seq(params, len(seq_prompts))
seq_lora_requests = self._lora_request_to_seq(lora_request, len(seq_prompts)) seq_lora_requests = self._lora_request_to_seq(lora_request, len(seq_prompts))
seq_priority = self._priority_to_seq(priority, len(prompts)) seq_priority = self._priority_to_seq(priority, len(seq_prompts))
return self._render_and_add_requests( return self._render_and_add_requests(
prompts=( prompts=(
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment