Commit f5f9f42f authored by zhuwenwen's avatar zhuwenwen
Browse files

Merge branch 'v0.7.2-dev-wm' into 'v0.7.2-dev'

[fix]修复开启并行解码后,在极端测试情况下服务无响应问题

See merge request dcutoolkit/deeplearing/vllm!97
parents f5f65d24 7a23da92
...@@ -690,6 +690,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase): ...@@ -690,6 +690,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
hidden_states = hidden_states[ hidden_states = hidden_states[
torch.where(sampler_output.sampled_token_ids - torch.where(sampler_output.sampled_token_ids -
VLLM_INVALID_TOKEN_ID)[0]] VLLM_INVALID_TOKEN_ID)[0]]
if not skip_proposer:
if self.previous_hidden_states is None and len( if self.previous_hidden_states is None and len(
seq_group_meta_with_hidden): seq_group_meta_with_hidden):
self.previous_hidden_states = HiddenStates( self.previous_hidden_states = HiddenStates(
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment