Skip to content

GitLab

Unverified Commit 31f46a0d authored by Nick Hill's avatar Nick Hill Committed by GitHub
Browse files

[BugFix] Fix seeded random sampling with encoder-decoder models (#8870) 


Co-authored-by: default avatarRoger Wang <ywang@roblox.com>
parent 3d49776b
Hide whitespace changes
Inline Side-by-side
Showing with 3 additions and 1 deletion
vllm/worker/enc_dec_model_runner.py
View file @ 31f46a0d
...@@ -268,11 +268,13 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]): ...@@ -268,11 +268,13 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
encoder_input_positions=encoder_input_positions_tensor, encoder_input_positions=encoder_input_positions_tensor,
) )
generators = self.get_generators(finished_requests_ids)
sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list, sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list,
model_input.seq_lens, model_input.seq_lens,
model_input.query_lens, model_input.query_lens,
self.device, self.device,
self.pin_memory) self.pin_memory,
generators=generators)
is_prompt = (seq_group_metadata_list[0].is_prompt is_prompt = (seq_group_metadata_list[0].is_prompt
if seq_group_metadata_list else None) if seq_group_metadata_list else None)
return dataclasses.replace(model_input, return dataclasses.replace(model_input,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment