"vscode:/vscode.git/clone" did not exist on "d460a18fc656f7fb217b977d4c2ee1003af2a5b6"
Unverified Commit 31f46a0d authored by Nick Hill's avatar Nick Hill Committed by GitHub
Browse files

[BugFix] Fix seeded random sampling with encoder-decoder models (#8870)


Co-authored-by: default avatarRoger Wang <ywang@roblox.com>
parent 3d49776b
......@@ -268,11 +268,13 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
encoder_input_positions=encoder_input_positions_tensor,
)
generators = self.get_generators(finished_requests_ids)
sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list,
model_input.seq_lens,
model_input.query_lens,
self.device,
self.pin_memory)
self.pin_memory,
generators=generators)
is_prompt = (seq_group_metadata_list[0].is_prompt
if seq_group_metadata_list else None)
return dataclasses.replace(model_input,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment