Unverified Commit 41273fd7 authored by Yusong Gao's avatar Yusong Gao Committed by GitHub
Browse files

fix: handle None multimodal_inputs during merging and filtering batches in...

fix: handle None multimodal_inputs during merging and filtering batches in disaggregation decode mode (#6169)
parent e9bebafb
...@@ -1485,7 +1485,8 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): ...@@ -1485,7 +1485,8 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
self.encoder_lens_cpu = [self.encoder_lens_cpu[i] for i in keep_indices] self.encoder_lens_cpu = [self.encoder_lens_cpu[i] for i in keep_indices]
self.reqs = [self.reqs[i] for i in keep_indices] self.reqs = [self.reqs[i] for i in keep_indices]
self.multimodal_inputs = [self.multimodal_inputs[i] for i in keep_indices] if self.multimodal_inputs is not None:
self.multimodal_inputs = [self.multimodal_inputs[i] for i in keep_indices]
self.req_pool_indices = self.req_pool_indices[keep_indices_device] self.req_pool_indices = self.req_pool_indices[keep_indices_device]
self.seq_lens = self.seq_lens[keep_indices_device] self.seq_lens = self.seq_lens[keep_indices_device]
self.out_cache_loc = None self.out_cache_loc = None
...@@ -1534,7 +1535,8 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): ...@@ -1534,7 +1535,8 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
self.top_logprobs_nums = [0] * len(self.reqs) + other.top_logprobs_nums self.top_logprobs_nums = [0] * len(self.reqs) + other.top_logprobs_nums
self.token_ids_logprobs = [None] * len(self.reqs) + other.token_ids_logprobs self.token_ids_logprobs = [None] * len(self.reqs) + other.token_ids_logprobs
self.reqs.extend(other.reqs) self.reqs.extend(other.reqs)
self.multimodal_inputs.extend(other.multimodal_inputs) if self.multimodal_inputs is not None:
self.multimodal_inputs.extend(other.multimodal_inputs)
self.return_logprob |= other.return_logprob self.return_logprob |= other.return_logprob
self.has_stream |= other.has_stream self.has_stream |= other.has_stream
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment