diff --git a/3rdparty/fetch_content.json b/3rdparty/fetch_content.json index 6ded1e29d280..4f7e1506b16d 100644 --- a/3rdparty/fetch_content.json +++ b/3rdparty/fetch_content.json @@ -62,7 +62,9 @@ "git_repository": "https://github.com/deepseek-ai/FlashMLA.git", "git_tag": "15f13e5030374295491c5ce31b02d7e63a7772c6", "git_submodules_recurse": true, - "source_subdir": "dont-add-this-project-with-add-subdirectory" + "source_subdir": "dont-add-this-project-with-add-subdirectory", + "patch_file": "patches/flashmla_release_gil.patch", + "_comment": "When changing git_tag, check the conditions for releasing the GIL that the comment in patches/flashmla_release_gil.patch lists against the new sources; the patch still applying does not show that they hold." }, { "name": "fmt", diff --git a/3rdparty/patches/flashmla_release_gil.patch b/3rdparty/patches/flashmla_release_gil.patch new file mode 100644 index 000000000000..080a7bfc70f0 --- /dev/null +++ b/3rdparty/patches/flashmla_release_gil.patch @@ -0,0 +1,52 @@ +--- a/csrc/api/api.cpp ++++ b/csrc/api/api.cpp +@@ -7,9 +7,44 @@ + + PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.doc() = "FlashMLA"; +- m.def("sparse_decode_fwd", &sparse_attn_decode_interface); +- m.def("dense_decode_fwd", &dense_attn_decode_interface); +- m.def("sparse_prefill_fwd", &sparse_attn_prefill_interface); +- m.def("dense_prefill_fwd", &FMHACutlassSM100FwdRun); +- m.def("dense_prefill_bwd", &FMHACutlassSM100BwdRun); ++ // Every binding releases the GIL for the duration of its C++ call. ++ // ++ // A binding that kept the GIL could deadlock a process that also queues Python host functions on a ++ // CUDA stream, as TensorRT-LLM does. What is seen then: under load, the thread that runs the model ++ // never returns from a call into this module, and in a multi-rank deployment the other ranks stall ++ // in their next collective. ++ // ++ // Three facts make the cycle: ++ // 1. A kernel launch is normally asynchronous, but once the launch queue is full it blocks until ++ // the device has worked through part of the queue. ++ // 2. A host function queued with cudaLaunchHostFunc blocks the work queued behind it until it has ++ // returned, and one that calls back into Python must take the GIL before it can run. ++ // 3. pybind11 keeps the GIL for the whole call of a function bound with a plain m.def, also while ++ // a launch inside it blocks. ++ // So when a launch made from here blocks on a full queue that has a Python host function ahead of ++ // it, the launch waits for the host function and the host function waits for the GIL. ++ // ++ // In TensorRT-LLM the host functions come from tensorrt_llm/_torch/hostfunc.py; the capturable ++ // guided decoder (guided decoding beside one-model speculative decoding) queues them for every ++ // step. Those captured into a CUDA graph run when a replay reaches them, which is after the Python ++ // call that started the replay has returned, so they can still be pending when the next step runs ++ // eagerly and calls into this module. Whether the launch queue is full at that moment depends on ++ // the load, so the hang shows up under serving load and not on every request. ++ // ++ // Releasing the GIL is safe for the functions bound here: their arguments and results are tensors ++ // and scalars, which pybind11 converts before the guard is constructed and after it is destroyed; ++ // they never call into Python; and they keep no mutable state between calls, so calls from ++ // different Python threads need no serialization by the GIL. A binding that takes or returns ++ // Python objects, calls into Python, or shares mutable state must not get this guard. ++ // ++ // These conditions were checked for the FlashMLA commit pinned in 3rdparty/fetch_content.json, and ++ // they are properties of that commit only. Whenever the pin moves, check them again for every ++ // binding of the new version and for the functions they bind. The patch still applying does not ++ // show that they hold: a binding added upstream gets no guard (and can deadlock as described above ++ // if it launches kernels), and a change inside a bound function leaves this file untouched. ++ m.def("sparse_decode_fwd", &sparse_attn_decode_interface, pybind11::call_guard()); ++ m.def("dense_decode_fwd", &dense_attn_decode_interface, pybind11::call_guard()); ++ m.def("sparse_prefill_fwd", &sparse_attn_prefill_interface, pybind11::call_guard()); ++ m.def("dense_prefill_fwd", &FMHACutlassSM100FwdRun, pybind11::call_guard()); ++ m.def("dense_prefill_bwd", &FMHACutlassSM100BwdRun, pybind11::call_guard()); + } diff --git a/tensorrt_llm/_torch/pyexecutor/adp_iter_stats.py b/tensorrt_llm/_torch/pyexecutor/adp_iter_stats.py index 45862bcc78df..129f96293d82 100644 --- a/tensorrt_llm/_torch/pyexecutor/adp_iter_stats.py +++ b/tensorrt_llm/_torch/pyexecutor/adp_iter_stats.py @@ -12,11 +12,12 @@ IterationStats, KvCacheStats, RequestStats, + SpecDecodingStats, ) from tensorrt_llm.logger import logger from .scheduler.adp_router import RankIterStatsPayload, RankState -from .scheduler.rank_state import RankKvCacheStats +from .scheduler.rank_state import RankKvCacheStats, RankSpecDecodingStats _ITERATION_STATS_SCALAR_FIELDS = ( "timestamp", @@ -33,10 +34,7 @@ "pinned_mem_usage", ) -_ITERATION_STATS_OPTIONAL_FIELDS = ( - "static_batching_stats", - "specdec_stats", -) +_ITERATION_STATS_OPTIONAL_FIELDS = ("static_batching_stats",) def _snapshot_kv_cache_stats(stats: KvCacheStats | None) -> RankKvCacheStats | None: @@ -78,6 +76,51 @@ def _restore_kv_cache_stats(snapshot: RankKvCacheStats) -> KvCacheStats: return stats +def _snapshot_specdec_stats(stats: SpecDecodingStats | None) -> RankSpecDecodingStats | None: + """Capture a rank's speculative-decoding counters. + + Args: + stats: Local speculative-decoding stats, or None when they are unavailable. + + Returns: + Immutable counter snapshot, or None when unavailable. + """ + if stats is None: + return None + return RankSpecDecodingStats( + num_draft_tokens=stats.num_draft_tokens, + num_accepted_tokens=stats.num_accepted_tokens, + num_requests_with_draft_tokens=stats.num_requests_with_draft_tokens, + ) + + +def _restore_specdec_stats( + snapshot: RankSpecDecodingStats, rank0_stats: SpecDecodingStats | None +) -> SpecDecodingStats: + """Create a native speculative-decoding stats object for one rank's row. + + The counters and the acceptance length derived from them are the rank's + own. The draft timings keep rank 0's values, like the row's other timings. + + Args: + snapshot: Counters captured with the rank's iteration payload. + rank0_stats: Rank 0's speculative-decoding stats for the same iteration. + + Returns: + A fresh native stats object. + """ + stats = SpecDecodingStats() + for name, value in zip(snapshot._fields, snapshot): + setattr(stats, name, value) + num_requests = snapshot.num_requests_with_draft_tokens + stats.acceptance_length = ( + (snapshot.num_accepted_tokens + num_requests) / num_requests if num_requests else 0.0 + ) + stats.iter_latency_ms = rank0_stats.iter_latency_ms if rank0_stats is not None else 0.0 + stats.draft_overhead = rank0_stats.draft_overhead if rank0_stats is not None else 0.0 + return stats + + @dataclass class ADPIterStatsRecord: """Append-ready stats row produced by Attention-DP fanout.""" @@ -141,6 +184,7 @@ def make_payload(stats: IterationStats) -> RankIterStatsPayload: num_paused_kv_tokens=ifb.num_paused_kv_tokens, kv_cache_stats=_snapshot_kv_cache_stats(stats.kv_cache_stats), cross_kv_cache_stats=_snapshot_kv_cache_stats(stats.cross_kv_cache_stats), + specdec_stats=_snapshot_specdec_stats(stats.specdec_stats), ) def queue( @@ -266,6 +310,11 @@ def _make_rank_iter_stats( if nested_stats is not None: setattr(stats, attr, nested_stats) + if payload.specdec_stats is not None: + stats.specdec_stats = _restore_specdec_stats( + payload.specdec_stats, rank0_stats.specdec_stats + ) + ifb = InflightBatchingStats() ifb.num_context_requests = payload.num_context_requests ifb.num_ctx_tokens = payload.num_ctx_tokens diff --git a/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/warmup.py b/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/warmup.py index e4763d201dd1..fd9bca875200 100644 --- a/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/warmup.py +++ b/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/warmup.py @@ -437,6 +437,17 @@ def _warmup_scheduled( generation_only = self.mapping.has_cp_helix() or is_decode_only( self.sparse_attention_config ) + # A capturable guided decoder rules out the general warmup's generation + # requests only: the autotuner below runs its context step with the + # decoder installed. Run the context requests anyway, so that the + # autotuner's max-token step is not the engine's first forward (with + # the buffers it allocates lazily) and runs after the release below, + # as it does without guided decoding. + can_run_context_warmup = can_run_general_warmup or self._agreement.agree_flag( + self._context_warmups_supported + and not generation_only + and not isinstance(kv_cache_manager, MambaHybridCacheManager) + ) log_mem_snapshot("warmup/before_warmup") # Compile the DSv4 indexer-Q CuTe DSL kernels before the first @@ -462,7 +473,7 @@ def _warmup_scheduled( resource_manager, can_run_general_warmup and not generation_only ) - if can_run_general_warmup: + if can_run_context_warmup: # Specialize torch.compile graphs across the key input shapes before CUDA graph capture. with self._warmup_timer.phase( "general", metrics=self._metrics, metric_name="general_warmup_seconds" @@ -472,6 +483,12 @@ def _warmup_scheduled( warmup_requests_configs = self._generation_warmup_shapes( warmup_requests_configs ) + elif not can_run_general_warmup: + warmup_requests_configs = [ + (num_tokens, num_gen_tokens) + for num_tokens, num_gen_tokens in warmup_requests_configs + if num_gen_tokens == 0 + ] warmup_requests_configs = self._agreement.agree_shapes(warmup_requests_configs) # Currently graph has not been captured, disable cuda graph for this warmup. with self._graphs.no_cuda_graph(): @@ -987,6 +1004,10 @@ def _general_warmup_impl( # Peers are inside the same forward's collectives and # cannot follow a rank-local skip. raise + if self.guided_decoder is not None: + # Input preparation enqueues the batch before model forward. + # An OOM may leave it unconsumed, or leave callbacks pending. + self.guided_decoder.reset_warmup_state() logger.warning( f"OOM during general warmup with {num_tokens} tokens, " f"{num_gen_tokens} generation tokens. Skipping." diff --git a/tensorrt_llm/_torch/pyexecutor/guided_decoder.py b/tensorrt_llm/_torch/pyexecutor/guided_decoder.py index 109d1b197cf7..a8f8fcfe4ae3 100644 --- a/tensorrt_llm/_torch/pyexecutor/guided_decoder.py +++ b/tensorrt_llm/_torch/pyexecutor/guided_decoder.py @@ -581,6 +581,20 @@ def _drain_host_functions(self) -> None: if not torch.cuda.is_current_stream_capturing(): self.bitmask_event.synchronize() + def reset_warmup_state(self) -> None: + """Discard abandoned dummy batches after an eager warmup failure. + + Only call during warmup, with no live requests or graph capture. Wait + on the stream itself: a failed forward may have enqueued callbacks + without reaching the bitmask-event record. Once they finish, no + callback can consume or overwrite the cleared request snapshots. + Warmup dummies have no grammar matchers to roll back. + """ + self.stream.synchronize() + self.queue = Queue() + self.requests = None + self.requests_hostfunc = None + @nvtx_range("GuidedDecoder.add_batch") def add_batch(self, scheduled_requests: ScheduledRequests, diff --git a/tensorrt_llm/_torch/pyexecutor/kv_cache/kv_cache_manager_v2.py b/tensorrt_llm/_torch/pyexecutor/kv_cache/kv_cache_manager_v2.py index f12801b255e1..edb9202addd4 100644 --- a/tensorrt_llm/_torch/pyexecutor/kv_cache/kv_cache_manager_v2.py +++ b/tensorrt_llm/_torch/pyexecutor/kv_cache/kv_cache_manager_v2.py @@ -6882,6 +6882,17 @@ def _prepare_draft_resources(self, scheduled_batch: ScheduledRequests): self._after_shrink(req.py_request_id, kv_cache) for req in scheduled_batch.generation_requests: + # A paired pool's generation dummy has a fixed shape, and + # add_dummy_requests reserves its full forward capacity in both + # pools, even when it joins the batch after scheduler admission. + # An unpaired pool is not handed to add_dummy_requests: the + # mirror the scheduler admits for its dummy is empty and grows + # below like any other. + preallocated_dummy = ( + self.enable_joint_kv_cache_reuse + and req.is_dummy_request + and req.py_request_id in self.kv_cache_map + ) kv_cache = self._mirror_draft_kv_cache(req) if kv_cache is None: raise RuntimeError( @@ -6898,7 +6909,7 @@ def _prepare_draft_resources(self, scheduled_batch: ScheduledRequests): ) # A paired scheduler already grew both pools atomically; the # recorded width says so, so do not charge this step twice. - if req.py_request_id in self._allocated_draft_lens: + if preallocated_dummy or req.py_request_id in self._allocated_draft_lens: continue new_cap = self._required_gen_capacity(req, kv_cache.capacity) if not kv_cache.resize(new_cap): diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index cd5703847df2..0ef8c9209f2d 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -5248,13 +5248,14 @@ def _finalize_adp_dummy_allocation(self, can_queue: bool) -> None: if spec_resource_manager is not None: spec_resource_manager.free_resources(dummy_request) self.kv_cache_manager.free_resources(dummy_request) - # The V2 scheduler also admits the draft pool's mirror while it - # schedules the dummy (KVCacheV2Scheduler._admit_unpaired_draft), so - # that pool holds an empty cache under the dummy ID too. Left behind, - # it outlives this dummy, and the deferred previous batch -- which - # still holds the earlier, finished dummy with the same ID -- resizes - # it to that dummy's history in update_resources ("History length - # cannot exceed capacity"). + # The draft pool holds a cache under the dummy ID too: a paired pool + # gets it with the dummy (add_dummy_requests), and the V2 scheduler + # admits an unpaired pool's mirror while it schedules the dummy + # (KVCacheV2Scheduler._admit_unpaired_draft). Left behind, it outlives + # this dummy, and the deferred previous batch -- which still holds the + # earlier, finished dummy with the same ID -- resizes it to that + # dummy's history in update_resources ("History length cannot exceed + # capacity"). draft_kv_cache_manager = getattr(self, "draft_kv_cache_manager", None) if (getattr(self, "_is_kv_manager_v2", False) and draft_kv_cache_manager is not None): @@ -9589,6 +9590,11 @@ def _pad_attention_dp_dummy_request(self): key="attention_dp_dummy_insufficient_kv_capacity") return + # A paired draft pool admits every scheduled request together with the + # target pool, so the dummy needs a cache in both. + draft_kv_cache_manager = (self.draft_kv_cache_manager + if self.enable_joint_kv_cache_reuse else None) + if (not self._enable_adp_dummy_fixes or self.kv_cache_transceiver is None): try: @@ -9597,6 +9603,7 @@ def _pad_attention_dp_dummy_request(self): token_nums=token_nums, is_gen=self._adp_dummy_is_gen, max_num_draft_tokens=self.max_total_draft_tokens, + draft_kv_cache_manager=draft_kv_cache_manager, ) except OutOfPagesError: dummy_requests = None @@ -9615,7 +9622,13 @@ def _pad_attention_dp_dummy_request(self): spec_resource_manager = self.resource_manager.get_resource_manager( ResourceManagerType.SPEC_RESOURCE_MANAGER) if spec_resource_manager is not None: - spec_resource_manager.add_dummy_requests(dummy_request_ids) + try: + spec_resource_manager.add_dummy_requests(dummy_request_ids) + except NoFreeSlotsError: + self.kv_cache_manager.free_resources(llm_request) + if draft_kv_cache_manager is not None: + draft_kv_cache_manager.free_resources(llm_request) + return self.active_requests.append(llm_request) return @@ -9632,6 +9645,7 @@ def _pad_attention_dp_dummy_request(self): token_nums=token_nums, is_gen=self._adp_dummy_is_gen, max_num_draft_tokens=self.max_total_draft_tokens, + draft_kv_cache_manager=draft_kv_cache_manager, ) except OutOfPagesError: dummy_requests = None @@ -9648,6 +9662,8 @@ def _pad_attention_dp_dummy_request(self): spec_resource_manager.add_dummy_requests(dummy_request_ids) except NoFreeSlotsError: self.kv_cache_manager.free_resources(dummy_request) + if draft_kv_cache_manager is not None: + draft_kv_cache_manager.free_resources(dummy_request) return assert dummy_request is not None @@ -9723,6 +9739,11 @@ def _pad_empty_attention_dp_batch( return dummy_request_ids = [ATTENTION_DP_DUMMY_REQUEST_ID] + # Allocate the paired draft cache with the dummy, where a shortage + # degrades to an empty batch, rather than leaving it to the draft + # manager's prepare_resources mirror, which raises on one. + draft_kv_cache_manager = (self.draft_kv_cache_manager + if self.enable_joint_kv_cache_reuse else None) try: # Degrade to an empty batch rather than propagate: the ranks have # yet to agree on can_queue, so a rank-local raise would strand the @@ -9733,6 +9754,7 @@ def _pad_empty_attention_dp_batch( is_gen=not context_dummy, max_num_draft_tokens=(0 if context_dummy else self.max_total_draft_tokens), + draft_kv_cache_manager=draft_kv_cache_manager, ) except (OutOfPagesError, NoFreeSlotsError): dummy_requests = None @@ -9751,6 +9773,8 @@ def _pad_empty_attention_dp_batch( spec_resource_manager.add_dummy_requests(dummy_request_ids) except NoFreeSlotsError: self.kv_cache_manager.free_resources(dummy_request) + if draft_kv_cache_manager is not None: + draft_kv_cache_manager.free_resources(dummy_request) return dummy_request.is_attention_dp_dummy = True diff --git a/tensorrt_llm/_torch/pyexecutor/scheduler/rank_state.py b/tensorrt_llm/_torch/pyexecutor/scheduler/rank_state.py index b6c1e4ab21c6..d9c40995f8ed 100644 --- a/tensorrt_llm/_torch/pyexecutor/scheduler/rank_state.py +++ b/tensorrt_llm/_torch/pyexecutor/scheduler/rank_state.py @@ -59,6 +59,40 @@ def deserialize(cls, data: list[int | float]) -> RankKvCacheStats: ) +class RankSpecDecodingStats(NamedTuple): + """Immutable rank-local speculative-decoding counters for one iteration.""" + + num_draft_tokens: int + num_accepted_tokens: int + num_requests_with_draft_tokens: int + + @classmethod + def wire_size(cls) -> int: + """Return the number of scalar fields in a speculative-decoding snapshot. + + Returns: + Number of fields in the named tuple's wire schema. + """ + return len(cls._fields) + + @classmethod + def deserialize(cls, data: list[int | float]) -> RankSpecDecodingStats: + """Restore the counters from their scalar wire fields. + + Args: + data: Draft tokens, accepted draft tokens and requests with draft tokens. + + Returns: + The immutable speculative-decoding snapshot. + + Raises: + ValueError: The snapshot has an unexpected number of fields. + """ + if len(data) != cls.wire_size(): + raise ValueError(f"Speculative-decoding snapshot requires {cls.wire_size()} fields") + return cls(int(data[0]), int(data[1]), int(data[2])) + + @dataclass class RankIterStatsPayload: """Per-rank iteration payload piggybacked on the ADP allgather.""" @@ -66,8 +100,8 @@ class RankIterStatsPayload: # Legacy scalar widths are independent of the optional snapshot fields. _ITER_STATS_FIELD_COUNT: ClassVar[int] = 9 _KV_LOAD_FIELD_COUNT: ClassVar[int] = 3 - _SNAPSHOT_WIRE_VERSION: ClassVar[int] = -1 - _SNAPSHOT_HEADER_SIZE: ClassVar[int] = 4 + _SNAPSHOT_WIRE_VERSION: ClassVar[int] = -2 + _SNAPSHOT_HEADER_SIZE: ClassVar[int] = 5 has_iter_stats: int = 0 iter_stats_iter: int = -1 @@ -83,6 +117,7 @@ class RankIterStatsPayload: kv_load_timestamp_ns: int = 0 kv_cache_stats: RankKvCacheStats | None = None cross_kv_cache_stats: RankKvCacheStats | None = None + specdec_stats: RankSpecDecodingStats | None = None def serialize(self, include_kv_cache_load: bool = False) -> list[int | float]: """Serialize to a flat list for allgather transport.""" @@ -105,17 +140,17 @@ def serialize(self, include_kv_cache_load: bool = False) -> list[int | float]: self.kv_load_timestamp_ns, ] ) - if self.kv_cache_stats is not None or self.cross_kv_cache_stats is not None: + snapshots = (self.kv_cache_stats, self.cross_kv_cache_stats, self.specdec_stats) + if any(snapshot is not None for snapshot in snapshots): # The negative version cannot be confused with has_iter_stats (0/1). - # Presence flags frame both snapshots and the independent load fields. + # Presence flags frame the snapshots and the independent load fields. values = [ self._SNAPSHOT_WIRE_VERSION, int(include_kv_cache_load), - int(self.kv_cache_stats is not None), - int(self.cross_kv_cache_stats is not None), + *(int(snapshot is not None) for snapshot in snapshots), *values, ] - for snapshot in (self.kv_cache_stats, self.cross_kv_cache_stats): + for snapshot in snapshots: if snapshot is not None: values.extend(snapshot) return values @@ -138,12 +173,13 @@ def snapshot_payload_size(cls, data: list[int | float]) -> int: or data[0] != cls._SNAPSHOT_WIRE_VERSION or any(flag not in (0, 1) for flag in data[1 : cls._SNAPSHOT_HEADER_SIZE]) ): - raise ValueError("Invalid KV snapshot payload header") + raise ValueError("Invalid snapshot payload header") return ( cls._SNAPSHOT_HEADER_SIZE + cls._ITER_STATS_FIELD_COUNT + int(data[1]) * cls._KV_LOAD_FIELD_COUNT + int(data[2] + data[3]) * RankKvCacheStats.wire_size() + + int(data[4]) * RankSpecDecodingStats.wire_size() ) @classmethod @@ -153,7 +189,7 @@ def deserialize(cls, data: list[int | float]) -> RankIterStatsPayload: if values and values[0] < 0: expected_size = cls.snapshot_payload_size(values) if len(values) != expected_size: - raise ValueError(f"KV snapshot payload requires {expected_size} fields") + raise ValueError(f"Snapshot payload requires {expected_size} fields") scalar_end = ( cls._SNAPSHOT_HEADER_SIZE + cls._ITER_STATS_FIELD_COUNT @@ -170,6 +206,11 @@ def deserialize(cls, data: list[int | float]) -> RankIterStatsPayload: result.cross_kv_cache_stats = RankKvCacheStats.deserialize( values[cursor : cursor + RankKvCacheStats.wire_size()] ) + cursor += RankKvCacheStats.wire_size() + if values[4]: + result.specdec_stats = RankSpecDecodingStats.deserialize( + values[cursor : cursor + RankSpecDecodingStats.wire_size()] + ) return result scalar_field_count = cls._ITER_STATS_FIELD_COUNT + cls._KV_LOAD_FIELD_COUNT if len(values) > scalar_field_count: @@ -225,7 +266,7 @@ def deserialize(cls, data: list[int | float]) -> RankState: values[prefix_field_count:] ) if len(values) != prefix_field_count + stats_field_count + 4: - raise ValueError("KV snapshot rank state requires four trailing fields") + raise ValueError("Snapshot rank state requires four trailing fields") else: # Legacy messages have optional load telemetry and two or four tail # fields. Keep their framing independent of the Python dataclass size. diff --git a/tests/unittest/_torch/disaggregation/test_benchmark_disagg.py b/tests/unittest/_torch/disaggregation/test_benchmark_disagg.py index b30177e29839..5dda01577ca5 100644 --- a/tests/unittest/_torch/disaggregation/test_benchmark_disagg.py +++ b/tests/unittest/_torch/disaggregation/test_benchmark_disagg.py @@ -755,6 +755,7 @@ def __init__( self.num_fetch_requests = num_fetch_requests self.benchmark_req_queues_size = benchmark_req_queues_size self.max_total_draft_tokens = 0 + self.enable_joint_kv_cache_reuse = False self._adp_dummy_is_gen = True self._pending_adp_dummy_request = None self._enable_adp_dummy_fixes = True diff --git a/tests/unittest/_torch/pyexecutor/engine/runners/decoder/test_warmup.py b/tests/unittest/_torch/pyexecutor/engine/runners/decoder/test_warmup.py index 87ee1978acec..685e974b0cf8 100644 --- a/tests/unittest/_torch/pyexecutor/engine/runners/decoder/test_warmup.py +++ b/tests/unittest/_torch/pyexecutor/engine/runners/decoder/test_warmup.py @@ -11,6 +11,7 @@ import unittest from collections.abc import Iterator from dataclasses import replace +from queue import Queue from types import ModuleType, SimpleNamespace from typing import TYPE_CHECKING from unittest import mock @@ -44,6 +45,7 @@ from tensorrt_llm._torch.pyexecutor.engine.runners.decoder.graphs import DecoderCudaGraphs from tensorrt_llm._torch.pyexecutor.engine.runners.decoder.metadata import DecoderMetadata from tensorrt_llm._torch.pyexecutor.engine.runners.decoder.warmup import DecoderWarmup +from tensorrt_llm._torch.pyexecutor.guided_decoder import CapturableGuidedDecoder from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import KVCacheManagerV2 from tensorrt_llm._torch.pyexecutor.resource_manager import ResourceManager, ResourceManagerType from tensorrt_llm._torch.pyexecutor.runtime_hooks import DEFAULT_RUNTIME_HOOKS @@ -1009,6 +1011,28 @@ def test_empty_cache_count_under_default(self) -> None: f"Expected exactly 2 empty_cache calls; got order={calls}", ) + def test_guided_decoder_runs_the_context_warmup_before_the_autotuner(self) -> None: + """A capturable guided decoder rules out the general warmup's + generation requests, not its context requests: those still run, then + the release of the MoE workspaces and the cache, then the autotuner.""" + model_engine, resource_manager = self.enterContext(_warmup_and_resources()) + model_engine._inputs.guided_decoder = Mock() + tracker = _Tracker() + calls, _ = _run_warmup_tracked(model_engine, resource_manager, tracker=tracker) + (((_, configs), _),) = tracker.call_args["general_warmup"] + self.assertTrue(configs, "Expected context warmup requests") + self.assertTrue( + all(num_gen_tokens == 0 for _, num_gen_tokens in configs), + f"Expected context requests only; got {configs}", + ) + self.assertIn((model_engine._config.max_num_tokens, 0), configs) + general_idx = calls.index("general_warmup") + self.assertEqual( + calls[general_idx : general_idx + 5], + ["general_warmup", "moe_clear", "empty_cache", "autotuner", "empty_cache"], + f"Expected the context warmup and its release before the autotuner; got {calls}", + ) + def test_helix_cp_runs_only_the_generation_autotuner_warmup(self) -> None: """With Helix CP, can_run_general_warmup is False and the context warmups (general, Mamba hybrid) are skipped; the autotuner warmup still @@ -1539,12 +1563,17 @@ def test_native_mxfp8_respects_disabled_global_autotuner(self) -> None: @pytest.mark.cpu_only +@pytest.mark.parametrize("has_guided_decoder", [False, True]) @pytest.mark.parametrize( "world_size,dwdp_size,is_fatal", [(1, 0, False), (2, 0, True), (1, 4, True)], ) -def test_general_warmup_oom_policy(world_size: int, dwdp_size: int, is_fatal: bool) -> None: +def test_general_warmup_oom_policy( + world_size: int, dwdp_size: int, is_fatal: bool, has_guided_decoder: bool +) -> None: engine = _general_warmup_engine(world_size=world_size, dwdp_size=dwdp_size) + guided_decoder = Mock() if has_guided_decoder else None + engine._inputs.guided_decoder = guided_decoder error = torch.OutOfMemoryError("asymmetric OOM") engine._forward_warmup = mock.Mock(side_effect=error if is_fatal else [error, None]) @@ -1560,12 +1589,115 @@ def test_general_warmup_oom_policy(world_size: int, dwdp_size: int, is_fatal: bo # The remaining shape is never attempted: this rank's peers are # already stuck in the failed forward's collectives. engine._forward_warmup.assert_called_once() + if guided_decoder is not None: + guided_decoder.reset_warmup_state.assert_not_called() else: assert engine._forward_warmup.call_count == 2 # A retry after an OOM between dispatch() and combine() has to start # from a clean MoE all-to-all state. engine._reset_moe_alltoall_state.assert_called_once_with() empty_cache.assert_called_once_with() + if guided_decoder is not None: + guided_decoder.reset_warmup_state.assert_called_once_with() + + +@pytest.mark.cpu_only +@pytest.mark.parametrize("callback_pending", [False, True]) +def test_general_warmup_oom_restores_guided_batch_queue(callback_pending: bool) -> None: + """A failed warmup must not shift the next request's grammar snapshot.""" + engine = _general_warmup_engine(world_size=1, dwdp_size=0) + decoder = object.__new__(CapturableGuidedDecoder) + decoder.queue = Queue() + decoder._num_added_batches = 0 + decoder.requests = None + decoder.requests_hostfunc = None + decoder.max_num_draft_tokens = 1 + engine._inputs.guided_decoder = decoder + pending_callbacks: list[Callable[[], None]] = [] + consumed: list[int] = [] + + def enqueue_callback(fn: Callable[..., None], *args: object, **kwargs: object) -> None: + pending_callbacks.append(lambda: fn(*args, **kwargs)) + + def synchronize() -> None: + while pending_callbacks: + pending_callbacks.pop(0)() + consumed.append(decoder.requests_hostfunc.requests[0].request_id) + + decoder.stream = Mock(synchronize=Mock(side_effect=synchronize)) + + def batch(request_id: int, *, guided: bool = False) -> SimpleNamespace: + request = SimpleNamespace( + py_request_id=request_id, + py_guided_decoding_params=object() if guided else None, + py_seq_slot=0, + py_batch_idx=None, + is_context_init_state=True, + is_last_context_chunk=True, + is_generation_in_progress_state=False, + llm_request_type=None, + py_decoding_iter=0, + get_last_tokens=lambda _: 1, + py_draft_tokens=[], + py_num_accepted_draft_tokens=0, + ) + return SimpleNamespace( + all_requests=lambda: [request], num_context_requests=1, num_generation_requests=0 + ) + + engine._dummy_requests = SimpleNamespace( + create_warmup_request=lambda _resources, tokens, _gen: batch(tokens), + release_batch_context=lambda requests, _resources: _released_batch(requests), + ) + + def forward(requests: SimpleNamespace, *_args: object, **_kwargs: object) -> None: + assert not pending_callbacks + assert decoder.queue.empty() + assert decoder.requests is None + assert decoder.requests_hostfunc is None + decoder.add_batch(requests) + if decoder.requests.requests[0].request_id == 128: + if callback_pending: + # OOM after submitting fetch_batch, before recording the + # completion event: recovery must drain the stream itself. + decoder.fetch_batch() + raise torch.OutOfMemoryError("warmup forward failed after enqueue") + decoder.fetch_batch() + decoder.stream.synchronize() + + engine._forward_warmup = forward + with ( + _no_cuda_side_effects(), + patch("tensorrt_llm._torch.hostfunc.launch_hostfunc", side_effect=enqueue_callback), + ): + engine._general_warmup_impl(object(), [(128, 0), (64, 0)]) + assert decoder.queue.empty() + assert consumed == ([128, 64] if callback_pending else [64]) + + decoder.add_batch(batch(1000, guided=True)) + decoder.fetch_batch() + decoder.stream.synchronize() + + assert consumed[-1] == 1000 + (guided_request,) = decoder.requests_hostfunc.valid_requests() + assert guided_request.require_matcher_init() + # The host functions hold the batch the host thread numbered last, so a + # build failure is reported under the real request's batch. + assert decoder.requests_hostfunc.batch_id == decoder.last_added_batch_id + assert decoder.queue.empty() + + +@pytest.mark.cpu_only +def test_general_warmup_aborts_if_guided_state_cannot_be_reset() -> None: + engine = _general_warmup_engine(world_size=1, dwdp_size=0) + engine._inputs.guided_decoder = Mock() + engine._inputs.guided_decoder.reset_warmup_state.side_effect = RuntimeError("drain failed") + engine._forward_warmup = Mock(side_effect=torch.OutOfMemoryError("warmup OOM")) + + with _no_cuda_side_effects(), pytest.raises(RuntimeError, match="drain failed"): + engine._general_warmup_impl(object(), [(128, 0), (64, 0)]) + + engine._forward_warmup.assert_called_once() @pytest.mark.cpu_only diff --git a/tests/unittest/_torch/pyexecutor/kv_cache/test_kvcm2_integration.py b/tests/unittest/_torch/pyexecutor/kv_cache/test_kvcm2_integration.py index 9ddd9429d3b9..dc83dea4758c 100644 --- a/tests/unittest/_torch/pyexecutor/kv_cache/test_kvcm2_integration.py +++ b/tests/unittest/_torch/pyexecutor/kv_cache/test_kvcm2_integration.py @@ -17,6 +17,7 @@ import os import sys from collections.abc import Callable +from contextlib import ExitStack from dataclasses import dataclass, field, replace from types import SimpleNamespace from unittest.mock import ANY, Mock, call, patch @@ -57,6 +58,7 @@ from tensorrt_llm._torch.pyexecutor.py_executor import PyExecutor from tensorrt_llm._torch.pyexecutor.resource_manager import ResourceManager, ResourceManagerType from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests +from tensorrt_llm._torch.pyexecutor.scheduler.scheduler_v2 import KVCacheV2Scheduler from tensorrt_llm._torch.speculative.dflash import DFlashSpecMetadata, DFlashWorker from tensorrt_llm.bindings import DataType, SamplingConfig from tensorrt_llm.bindings.BuildInfo import ENABLE_MULTI_DEVICE @@ -66,6 +68,7 @@ from tensorrt_llm.conversation_params import ConversationParams from tensorrt_llm.llmapi.llm_args import ( BlockReuseConfig, + CapacitySchedulerPolicy, DecodingBaseConfig, DFlashDecodingConfig, DraftTargetDecodingConfig, @@ -3349,6 +3352,276 @@ def test_generation_dummy_uses_available_capacity(draft_len: int) -> None: manager.shutdown() +def _adp_dummy_executor( + cleanup: ExitStack, + spec: DFlashDecodingConfig, + *, + joint_reuse: bool, + transactional: bool, +) -> tuple[PyExecutor, dict[ResourceManagerType, KVCacheManagerV2]]: + """An attention-DP executor over real target and draft KV pools. + + ``joint_reuse`` pairs the two pools (block reuse on both); without it the + draft pool is an unpaired mirror of the target pool. + """ + managers = {} + for resource_type, is_draft in ( + (ResourceManagerType.KV_CACHE_MANAGER, False), + (ResourceManagerType.DRAFT_KV_CACHE_MANAGER, True), + ): + manager = KVCacheManagerV2( + KvCacheConfig(enable_block_reuse=joint_reuse, max_gpu_total_bytes=4 << 20), + CacheType.SELF, + num_layers=1, + num_kv_heads=2, + head_dim=64, + tokens_per_block=32, + max_seq_len=128, + max_batch_size=4, + max_num_tokens=128, + mapping=Mapping(), + spec_config=spec, + is_draft=is_draft, + joint_kv_cache_reuse=joint_reuse, + ) + cleanup.callback(manager.shutdown) + managers[resource_type] = manager + + executor = object.__new__(PyExecutor) + executor.resource_manager = ResourceManager(managers) + executor.kv_cache_manager = managers[ResourceManagerType.KV_CACHE_MANAGER] + executor.draft_kv_cache_manager = managers[ResourceManagerType.DRAFT_KV_CACHE_MANAGER] + executor.scheduler = KVCacheV2Scheduler( + max_batch_size=4, + max_num_tokens=128, + kv_cache_manager=executor.kv_cache_manager, + draft_kv_cache_manager=executor.draft_kv_cache_manager, + scheduler_policy=CapacitySchedulerPolicy.MAX_UTILIZATION, + ) + executor.enable_attention_dp = True + # Two attention-DP ranks: a single-rank group skips dummy padding. + executor.dist = SimpleNamespace(tp_size=2) + executor.kv_cache_transceiver = object() if transactional else None + executor.adp_router = SimpleNamespace(exclude_retiring_requests=False) + executor.active_requests = [] + executor.expected_num_active_requests = 1 + executor.max_num_active_requests = 4 + executor.max_total_draft_tokens = spec.max_draft_len + executor._adp_dummy_is_gen = True + executor._pending_adp_dummy_request = None + executor._enable_adp_dummy_fixes = True + executor._enable_scheduler_aware_adp_dummy = False + executor._enable_non_overlap_adp_forward_intent = False + executor._benchmark_fill_phase_active = False + executor._is_kv_manager_v2 = True + executor.enable_joint_kv_cache_reuse = joint_reuse + return executor, managers + + +@pytest.mark.parametrize("draft_len", [3, 8]) +@pytest.mark.parametrize("path", ["plain", "transactional", "empty_batch"]) +def test_adp_dummy_holds_paired_draft_kv(path: str, draft_len: int) -> None: + """An idle attention-DP rank's dummy needs a cache in both paired pools. + + Under joint KV cache reuse the scheduler admits a generation request in + the target and the draft pool together, so a dummy holding only a target + cache is never scheduled and the fleet cannot run a forward. Each padding + path must allocate the dummy in both pools, and a vetoed iteration must + release both so the singleton dummy ID can be allocated again. + """ + if not torch.cuda.is_available(): + pytest.skip("requires CUDA") + torch.cuda.init() + spec = DFlashDecodingConfig(max_draft_len=draft_len, speculative_model="draft") + with ExitStack() as cleanup: + executor, managers = _adp_dummy_executor( + cleanup, spec, joint_reuse=True, transactional=path == "transactional" + ) + + def assert_dummy_in_both_pools(dummy: LlmRequest) -> None: + for manager in managers.values(): + assert manager.kv_cache_map[dummy.py_request_id].is_active + + def assert_dummy_released(dummy: LlmRequest) -> None: + assert executor.active_requests == [] + for manager in managers.values(): + assert dummy.py_request_id not in manager.kv_cache_map + assert dummy.py_request_id not in manager._allocated_draft_lens + assert dummy.py_request_id not in manager._inflight_draft_lens + + def prepare_without_growing(scheduled: ScheduledRequests, dummy: LlmRequest) -> None: + capacities = { + key: manager.kv_cache_map[dummy.py_request_id].capacity + for key, manager in managers.items() + } + executor.resource_manager.prepare_resources(scheduled) + for key, manager in managers.items(): + assert manager.kv_cache_map[dummy.py_request_id].capacity == capacities[key] + + if path == "empty_batch": + # The dummy joins the already scheduled batch directly; a vetoed + # iteration rolls it back, and the next one pads again. + for can_queue in (False, True): + scheduled = ScheduledRequests() + executor._pad_empty_attention_dp_batch(scheduled) + (dummy,) = scheduled.generation_requests + assert executor.active_requests == [dummy] + assert_dummy_in_both_pools(dummy) + executor._finalize_adp_dummy_allocation(can_queue) + if not can_queue: + assert_dummy_released(dummy) + else: + prepare_without_growing(scheduled, dummy) + executor.resource_manager.free_resources(dummy) + for manager in managers.values(): + assert dummy.py_request_id not in manager.kv_cache_map + return + + # A rejected fleet vote must release both pools and permit the same + # singleton dummy ID to be allocated and admitted on the next pass. + for attempt in range(2 if path == "transactional" else 1): + executor._pad_attention_dp_dummy_request() + (dummy,) = executor.active_requests + scheduled = executor.scheduler.schedule_request(executor.active_requests, set()) + assert scheduled.generation_requests == [dummy] + assert scheduled.paused_requests == [] + assert_dummy_in_both_pools(dummy) + if path == "transactional" and attempt == 0: + executor._revert_gen_alloc(scheduled) + executor._finalize_adp_dummy_allocation(False) + assert_dummy_released(dummy) + else: + executor._finalize_adp_dummy_allocation(True) + prepare_without_growing(scheduled, dummy) + executor.resource_manager.free_resources(dummy) + for manager in managers.values(): + assert dummy.py_request_id not in manager.kv_cache_map + + +@pytest.mark.parametrize("path", ["plain", "transactional", "empty_batch"]) +def test_adp_dummy_grows_unpaired_draft_mirror(path: str) -> None: + """An unpaired draft pool sizes the attention-DP dummy's mirror in prepare_resources. + + The dummy is created in the target pool only. The scheduler admits the + draft mirror while it schedules the dummy but leaves it empty, and the + empty-batch path leaves no mirror at all, so the draft manager has to grow + it by one generation step. An empty mirror cannot take the history of the + previous iteration's finished dummy, which shares its ID and which the + overlap loop applies once this iteration is prepared. + """ + if not torch.cuda.is_available(): + pytest.skip("requires CUDA") + torch.cuda.init() + spec = DFlashDecodingConfig(max_draft_len=8, speculative_model="draft") + with ExitStack() as cleanup: + executor, managers = _adp_dummy_executor( + cleanup, spec, joint_reuse=False, transactional=path == "transactional" + ) + draft = executor.draft_kv_cache_manager + assert executor.scheduler._joint_draft_manager is None + + previous_batch = None + for _ in range(2): + batch = ScheduledRequests() + if path == "empty_batch": + executor._pad_empty_attention_dp_batch(batch) + (dummy,) = batch.generation_requests + assert dummy.py_request_id not in draft.kv_cache_map + else: + executor._pad_attention_dp_dummy_request() + (dummy,) = executor.active_requests + scheduled = executor.scheduler.schedule_request(executor.active_requests, set()) + assert scheduled.generation_requests == [dummy] + batch.generation_requests.append(dummy) + # Admitted by the scheduler, not sized. + assert draft.kv_cache_map[dummy.py_request_id].capacity == 0 + executor._finalize_adp_dummy_allocation(True) + + executor.resource_manager.prepare_resources(batch) + draft_cache = draft.kv_cache_map[dummy.py_request_id] + assert draft_cache.capacity == draft._required_gen_capacity(dummy, 0) + + if previous_batch is not None: + # _process_previous_batch: the deferred batch still holds the + # finished dummy, whose ID now maps to this iteration's mirror. + executor.resource_manager.update_resources( + previous_batch, skip_kv_cache_manager=True + ) + # _terminate_attention_dp_dummy_requests. + dummy.state = LlmRequestState.GENERATION_COMPLETE + executor.resource_manager.free_resources(dummy) + executor.active_requests.remove(dummy) + previous_batch = batch + + for manager in managers.values(): + assert dummy.py_request_id not in manager.kv_cache_map + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "joint_reuse,is_dummy,mirror_capacity,grows", + [ + # Paired pool: add_dummy_requests sized the dummy's mirror with the dummy. + pytest.param(True, True, 24, False, id="paired-dummy-preallocated"), + # Unpaired pool: the scheduler admitted the dummy's mirror, still empty. + pytest.param(False, True, 0, True, id="unpaired-dummy-admitted-empty"), + # Unpaired pool, empty-batch padding: the dummy has no mirror yet. + pytest.param(False, True, None, True, id="unpaired-dummy-no-mirror"), + # Real requests are a control: they grow by one step in either pool. + pytest.param(True, False, 24, True, id="paired-request"), + pytest.param(False, False, 24, True, id="unpaired-request"), + pytest.param(False, False, None, True, id="unpaired-request-no-mirror"), + ], +) +def test_draft_mirror_preserves_preallocated_dummy_capacity( + joint_reuse: bool, is_dummy: bool, mirror_capacity: int | None, grows: bool +) -> None: + """A paired pool's preallocated dummy fits one page; charging another step would need two. + + Every other mirror, an unpaired pool's dummy included, is sized here. + """ + request = LlmRequest( + request_id=100, + max_new_tokens=1, + input_tokens=[1] * 9, + sampling_config=SamplingConfig(1), + is_streaming=False, + draft_tokens=[1] * 8, + ) + request.state = LlmRequestState.GENERATION_IN_PROGRESS + request.is_dummy_request = is_dummy + capacity = mirror_capacity or 0 + cache = Mock(capacity=capacity, is_active=True) + # Dummies have only one 32-token page available. Real requests are a + # control: unlike fixed-shape dummies they still need per-step growth. + cache.resize.side_effect = lambda capacity: capacity <= (32 if is_dummy else 64) + manager = object.__new__(KVCacheManagerV2) + manager.is_draft = True + manager.enable_joint_kv_cache_reuse = joint_reuse + manager._mirror_uses_draft_view = False + manager._kv_reserve_draft_tokens = 8 + manager._allocated_draft_lens = {} + manager.kv_cache_map = {} if mirror_capacity is None else {request.py_request_id: cache} + manager._mirror_draft_kv_cache = Mock(return_value=cache) + manager._resume_and_restore = Mock(return_value=True) + manager._ensure_generation_beam_width = Mock(return_value=True) + manager._mirror_resize_error = Mock(return_value=RuntimeError("draft pool exhausted")) + batch = ScheduledRequests() + batch.generation_requests.append(request) + + manager._prepare_draft_resources(batch) + + manager._resume_and_restore.assert_called_once_with(request.py_request_id, cache) + if grows: + # One generation step: the token itself and the eight draft slots. + cache.resize.assert_called_once_with(capacity + 9) + else: + cache.resize.assert_not_called() + # Recognizing a preallocated dummy must not fabricate scheduler rollback + # bookkeeping; the dummy is freed as a whole after the iteration. + assert manager._allocated_draft_lens == {} + + def test_generation_dummy_does_not_reuse_real_request_prefix() -> None: if not torch.cuda.is_available(): pytest.skip("requires CUDA") diff --git a/tests/unittest/_torch/pyexecutor/test_adp_router.py b/tests/unittest/_torch/pyexecutor/test_adp_router.py index 068206650ff4..edbb49fccc7c 100644 --- a/tests/unittest/_torch/pyexecutor/test_adp_router.py +++ b/tests/unittest/_torch/pyexecutor/test_adp_router.py @@ -25,6 +25,7 @@ import inspect import random +from dataclasses import replace from types import SimpleNamespace from unittest.mock import MagicMock, Mock @@ -55,7 +56,10 @@ GenAdmissionConfig, GenAdmissionWaitingQueue, ) -from tensorrt_llm._torch.pyexecutor.scheduler.rank_state import RankKvCacheStats +from tensorrt_llm._torch.pyexecutor.scheduler.rank_state import ( + RankKvCacheStats, + RankSpecDecodingStats, +) from tensorrt_llm._torch.pyexecutor.scheduler.waiting_queue import PriorityWaitingQueue from tensorrt_llm.bindings.executor import RequestType from tensorrt_llm.conversation_params import ConversationParams @@ -311,8 +315,13 @@ def test_kv_snapshot_roundtrip_preserves_tail( assert RankIterStatsPayload.deserialize(payload.serialize(include_kv_cache_load)) == payload assert RankState.deserialize(state.serialize(include_kv_cache_load)) == state - def test_kv_snapshot_wire_schema_is_stable(self) -> None: + def test_snapshot_wire_schema_is_stable(self) -> None: """Pin field order and the complete versioned frame independently of roundtrips.""" + assert RankSpecDecodingStats._fields == ( + "num_draft_tokens", + "num_accepted_tokens", + "num_requests_with_draft_tokens", + ) assert RankKvCacheStats._fields == ( "max_num_blocks", "free_num_blocks", @@ -344,6 +353,7 @@ def test_kv_snapshot_wire_schema_is_stable(self) -> None: kv_load_timestamp_ns=123, kv_cache_stats=RankKvCacheStats(100, 80, 20, 16, 40, 30, 7, 5, 0.375), cross_kv_cache_stats=RankKvCacheStats(60, 45, 15, 32, 25, 18, 4, 3, 0.25), + specdec_stats=RankSpecDecodingStats(15, 9, 5), ), num_context_requests=3, num_generation_requests=4, @@ -355,10 +365,11 @@ def test_kv_snapshot_wire_schema_is_stable(self) -> None: 7, 321, 2, # Rank state prefix. - -1, + -2, 1, 1, - 1, # Version and load/primary/cross presence flags. + 1, + 1, # Version and load/primary/cross/speculative-decoding presence flags. 1, 9, 2, @@ -389,6 +400,9 @@ def test_kv_snapshot_wire_schema_is_stable(self) -> None: 4, 3, 0.25, # Cross KV snapshot. + 15, + 9, + 5, # Speculative-decoding snapshot. 3, 4, 6, @@ -418,6 +432,43 @@ def test_kv_snapshot_invalid_framing_is_rejected(self) -> None: with pytest.raises(ValueError, match="requires"): RankIterStatsPayload.deserialize(state.iter_stats.serialize()[:-1]) + @pytest.mark.parametrize("include_kv_cache_load", [False, True]) + @pytest.mark.parametrize("include_kv_cache_stats", [False, True]) + def test_specdec_snapshot_roundtrip_preserves_tail( + self, include_kv_cache_load: bool, include_kv_cache_stats: bool + ) -> None: + """Carry rank-local speculative-decoding counters with or without a KV snapshot.""" + payload = RankIterStatsPayload( + has_iter_stats=1, + iter_stats_iter=7, + num_context_requests=2, + num_gen_requests=5, + kv_used_blocks=4 if include_kv_cache_load else -1, + kv_total_blocks=10 if include_kv_cache_load else -1, + kv_load_timestamp_ns=123 if include_kv_cache_load else 0, + kv_cache_stats=( + RankKvCacheStats(10, 8, 2, 16, 3, 2, 1, 1, 0.5) if include_kv_cache_stats else None + ), + specdec_stats=RankSpecDecodingStats(15, 9, 5), + ) + state = RankState( + rank=2, + num_active_requests=7, + num_active_tokens=100, + num_retiring_requests=1, + iter_stats=payload, + num_context_requests=2, + num_generation_requests=5, + num_connector_transfers=3, + connector_poll_failed=True, + ) + + assert RankIterStatsPayload.deserialize(payload.serialize(include_kv_cache_load)) == payload + assert RankState.deserialize(state.serialize(include_kv_cache_load)) == state + # Zero counters are a measurement and stay distinct from an absent snapshot. + idle = replace(payload, specdec_stats=RankSpecDecodingStats(0, 0, 0)) + assert RankIterStatsPayload.deserialize(idle.serialize(include_kv_cache_load)) == idle + class TestDefaultADPRouter: # Router tests model strict placement, relaxed placement, and capacity diff --git a/tests/unittest/_torch/pyexecutor/test_iter_stats_populate.py b/tests/unittest/_torch/pyexecutor/test_iter_stats_populate.py index 8afb1a0fed04..fef6e4aab1f6 100644 --- a/tests/unittest/_torch/pyexecutor/test_iter_stats_populate.py +++ b/tests/unittest/_torch/pyexecutor/test_iter_stats_populate.py @@ -940,6 +940,80 @@ def test_attention_dp_fanout_emits_rank_local_rows_with_rank0_queue(): assert buffer.next_payload() is None +@pytest.mark.parametrize("rank0_idle", [False, True]) +@pytest.mark.parametrize("include_kv_cache_load", [False, True]) +def test_attention_dp_specdec_fanout_uses_rank_local_counts( + rank0_idle: bool, include_kv_cache_load: bool +) -> None: + """Transport asymmetric real/dummy work without copying rank 0's AL.""" + real_requests = [ + [] if rank0_idle else [_StubRequest(num_draft_tokens=3, py_num_accepted_draft_tokens=2)], + [ + _StubRequest(num_draft_tokens=3, py_num_accepted_draft_tokens=0), + _StubRequest(num_draft_tokens=3, py_num_accepted_draft_tokens=1), + _StubRequest(num_draft_tokens=3, py_num_accepted_draft_tokens=2), + ], + ] + buffers = [] + rank_states = [] + for rank, requests in enumerate(real_requests): + dummies = [ + _StubRequest(py_draft_tokens=[1, 1, 1], is_attention_dp_dummy=True), + _StubRequest(py_draft_tokens=[1, 1, 1], is_cuda_graph_dummy=True), + ] + stats = _invoke_update_iter_stats( + _StubScheduledBatch(gen_reqs=requests + dummies), + [], + num_ctx_tokens=0, + specdec=True, + ) + stats.iter = 7 + buffer = _build_adp_stats_buffer(stats, is_rank0=rank == 0) + buffers.append(buffer) + state = RankState(rank=rank, iter_stats=buffer.next_payload()) + rank_states.append(RankState.deserialize(state.serialize(include_kv_cache_load))) + + records = buffers[0].finalize(rank_states, is_rank0=True) + assert len(records) == 2 + expected_counts = [(0, 0, 0) if rank0_idle else (3, 2, 1), (9, 3, 3)] + for record, (drafted, accepted, requests) in zip(records, expected_counts): + spec = record.stats.specdec_stats + assert spec is not None + assert spec.num_draft_tokens == drafted + assert spec.num_accepted_tokens == accepted + assert spec.num_requests_with_draft_tokens == requests + assert spec.acceptance_length == pytest.approx( + (accepted + requests) / requests if requests else 0.0 + ) + + accepted = sum(record.stats.specdec_stats.num_accepted_tokens for record in records) + requests = sum(record.stats.specdec_stats.num_requests_with_draft_tokens for record in records) + assert (accepted + requests) / requests == pytest.approx(2.0 if rank0_idle else 2.25) + + +def test_attention_dp_specdec_fanout_does_not_fill_synthetic_rank_from_rank0() -> None: + """An explicit zero payload must not inherit another rank's drafting.""" + stats = _invoke_update_iter_stats( + _StubScheduledBatch( + gen_reqs=[_StubRequest(num_draft_tokens=3, py_num_accepted_draft_tokens=2)] + ), + [], + num_ctx_tokens=0, + specdec=True, + ) + stats.iter = 7 + buffer = _build_adp_stats_buffer(stats) + records = buffer.finalize( + [ + RankState(rank=0, iter_stats=buffer.next_payload()), + RankState(rank=1, iter_stats=RankIterStatsPayload(has_iter_stats=1, iter_stats_iter=7)), + ], + is_rank0=True, + ) + assert records[0].stats.specdec_stats.num_requests_with_draft_tokens == 1 + assert records[1].stats.specdec_stats is None + + def test_attention_dp_fanout_waits_for_complete_matching_payloads(): """Verify ADP fanout waits until every rank reports the rank-0 iteration.""" rank0_stats = _make_adp_iteration_stats(iter_id=9, num_context_requests=1) @@ -1076,7 +1150,7 @@ def test_attention_dp_fanout_copy_lists_cover_iteration_stats_fields(): copied_fields = ( set(_ITERATION_STATS_SCALAR_FIELDS) | set(_ITERATION_STATS_OPTIONAL_FIELDS) - | {"inflight_batching_stats", "kv_cache_stats", "cross_kv_cache_stats"} + | {"inflight_batching_stats", "kv_cache_stats", "cross_kv_cache_stats", "specdec_stats"} ) assert copied_fields == actual_fields diff --git a/tests/unittest/_torch/pyexecutor/test_py_executor.py b/tests/unittest/_torch/pyexecutor/test_py_executor.py index cf8383290002..5e660220412a 100644 --- a/tests/unittest/_torch/pyexecutor/test_py_executor.py +++ b/tests/unittest/_torch/pyexecutor/test_py_executor.py @@ -3164,6 +3164,7 @@ def __init__( self.expected_num_active_requests = 1 self.max_num_active_requests = 8 self.max_total_draft_tokens = 0 + self.enable_joint_kv_cache_reuse = False self.max_num_tokens = max_num_tokens self._adp_dummy_is_gen = True self._pending_adp_dummy_request = None diff --git a/tests/unittest/_torch/test_connector_adp.py b/tests/unittest/_torch/test_connector_adp.py index c3cd2e1abcf2..f7d5fe59983c 100644 --- a/tests/unittest/_torch/test_connector_adp.py +++ b/tests/unittest/_torch/test_connector_adp.py @@ -116,6 +116,7 @@ def _configure_recovery_controls(executor: "PyExecutor") -> None: executor._profile_manager = MagicMock() executor.model_engine = SimpleNamespace(cuda_graph_runner=None) executor.draft_model_engine = None + executor.enable_joint_kv_cache_reuse = False executor.sampler = None executor.worker_thread = MagicMock() executor._shutdown_sleep_wakeup_listeners = MagicMock()