"docs/ZH_CN/source/method_tutorials/cache.md" did not exist on "5a4db4905e69be19b412bcfa25d9bcd5683a426c"
Unverified Commit b79f75fd authored by Yineng Zhang's avatar Yineng Zhang Committed by GitHub
Browse files

[Auto Sync] Update scheduler.py (20251017) (#11738)

parent 8fcc69e7
...@@ -2166,6 +2166,12 @@ class Scheduler( ...@@ -2166,6 +2166,12 @@ class Scheduler(
batch.prepare_for_decode() batch.prepare_for_decode()
return batch return batch
# placeholder for override
def update_cache_from_scheduler(
self, schedule_batch: ScheduleBatch, batch_result: GenerationBatchResult
):
pass
def run_batch( def run_batch(
self, batch: ScheduleBatch self, batch: ScheduleBatch
) -> Union[GenerationBatchResult, EmbeddingBatchResult]: ) -> Union[GenerationBatchResult, EmbeddingBatchResult]:
...@@ -2242,6 +2248,7 @@ class Scheduler( ...@@ -2242,6 +2248,7 @@ class Scheduler(
batch_or_worker_batch batch_or_worker_batch
) )
future_indices_or_next_token_ids = batch_result.next_token_ids future_indices_or_next_token_ids = batch_result.next_token_ids
self.update_cache_from_scheduler(batch, batch_result)
# NOTE: future_indices_or_next_token_ids is used in ScheduleBatch, # NOTE: future_indices_or_next_token_ids is used in ScheduleBatch,
# which can probably be replaced by future_indices later [TODO(lsyin)]. # which can probably be replaced by future_indices later [TODO(lsyin)].
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment