pause speculative decoding with zero overhead scheduling, develop tbo first

vllm/spec_decode/spec_decode_worker.py

@@ -208,6 +208,9 @@ class SpecDecodeWorker(LoRANotSupportedWorkerBase):
                 if draft_model_config.hf_config.model_type == "eagle":
                     enable_lm_head_weight_load = True
                 if is_zero_overhead():
+                    assert False, (
+                                "speculative decoding not support zero overhead scheduler yet"
+                            )
                     from vllm.zero_overhead.spec_decode.muti_step_worker import ZeroOverheadMultiStepWorker
                     proposer_worker = ZeroOverheadMultiStepWorker(**draft_worker_kwargs)
                 else:

vllm/zero_overhead/llm_engine.py

@@ -301,6 +301,7 @@ class ZeroOverheadEngine(LLMEngine):
                     ) = self.scheduler[virtual_engine].schedule()
                 if self.last_record is not None:
                     last_sampler = self.last_record[1]
+                    spec_step = get_spec_step()
                     if spec_step == SpecStepKind.KIND_DEFAULT:
                         self.async_d2h = last_sampler.sampled_token_ids_tensor.to('cpu', non_blocking=True)
                     elif spec_step == SpecStepKind.SCORE_DECODE: